seasharp/anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
427588df0f
anvil/tools/anvil-manage-files
Digimer 41cd1e0319 * Several bugs fixed and enhancements; 
* DRBD is now configured to a ping-timeout of 3 seconds.
* Created Log->switches() that returnes the command line switches used by Anvil! tool command line calls based on the active log levels / secure logging. Appended this to all invocations of our tools.
* Updated Database->resync_databases() to now only skip 'jobs' and 'variables' tables with less than 10 record differences. All other differences will trigger a resync.
* Created System->_check_anvil_conf() that, as you might guess, checks in anvil.conf exists and created it (using defaults), if not. It also checks to see if the 'admin' group and user exists and creates them, if not.
* Updated anvil-daemon to check anvil.conf on start up and in each loop. Created the function check_journald() that checks (and sets, if needed) that journald logging is persistent.
* Made striker-manage-peers to check_if_configured on the Database->connect() when updating anvil.conf and the target UUID is the local machine. Also created a loop to make the reconnection a lot more robust.

Signed-off-by: Digimer <digimer@alteeve.ca>
2021-05-24 00:09:32 -04:00

1241 lines
46 KiB
Perl
Executable File
Raw Blame History

#!/usr/bin/perl
#
# This handles moving around and managing files on Anvil! nodes, DR hosts and Striker dashboards.
#
# When this is called (periodically by the daemon of after an upload / ISO generation);
# - 1. Check 'incoming/' for files. For any found, generate an md5sum and see if the file name and sum match
# anything in the database from any host;
# - If so, update/add an entry in 'file_locations'
# - If not, create a new entry in 'files' and then add the first entry in 'file_locations'
# - 2. Check 'file_locations' for any files on this system, and verify they exist still.
# - If not, check the other files for one with a matching md5sum. If found, handle as a rename.
# - If not found at all, search for the file according to the search rules and copy to here if found.
# - If not found anywhere, remove it from 'file_locations' and send an alert.
# - If found, check the size. If it differs, recalculate the md5sum.
# - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If
# also used with '--everywhere', then all copies on all systems we know about will be deleted. This is
# done by registering a job against all known hosts. As such, if this is called and the target file
# doesn't exist, it just clears the job and then exits.
# - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit.
#
# Exit codes;
# 0 = Normal exit or md5sum of this program changed and it exited to reload.
# 1 = No databases available.
# 2 = Another process that is still running has picked up this job.
# 3 = No filename specified when needed by another switch.
# 4 = Request to change the file name but the new name wasn't given.
# 5 = The file specified was not found.
# 6 = The file to delete is not under '/mnt/shared/'.
#
# TODO:
# -
#
# NOTE:
# -
#
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
# Disable buffering
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
$anvil->data->{switches}{'delete'} = "";
$anvil->data->{switches}{download} = "";
$anvil->data->{switches}{everywhere} = "";
$anvil->data->{switches}{file} = "";
$anvil->data->{switches}{'is-script'} = "";
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'rename'} = "";
$anvil->data->{switches}{to} = "";
$anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"switches::delete" => $anvil->data->{switches}{'delete'},
"switches::download" => $anvil->data->{switches}{download},
"switches::everywhere" => $anvil->data->{switches}{everywhere},
"switches::file" => $anvil->data->{switches}{file},
"switches::is-script" => $anvil->data->{switches}{'is-script'},
"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'},
"switches::rename" => $anvil->data->{switches}{'rename'},
"switches::to" => $anvil->data->{switches}{to},
}});
# Connect or die
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
# If we have a job_uuid, pick it up.
if ($anvil->data->{switches}{'job-uuid'})
{
my $return = $anvil->Job->get_job_details({check => 1, job_uuid => $anvil->data->{switches}{'job-uuid'}});
if ($return == 1)
{
# It's not a UUID.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({exit_code => 2});
}
if ($return == 2)
{
# This job is being handled by another process that is still alive.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({exit_code => 3});
}
# If there's a progress, clear it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'job::job_progress' => $anvil->data->{job}{job_progress} }});
if ($anvil->data->{job}{job_progress})
{
$anvil->Job->update_progress({
progress => 1,
message => "clear",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
}
}
# What are we doing?
if ($anvil->data->{switches}{'rename'})
{
handle_rename($anvil);
}
elsif ($anvil->data->{switches}{'delete'})
{
handle_delete($anvil);
}
elsif ($anvil->data->{switches}{'is-script'})
{
handle_script($anvil);
}
else
{
# Check for files scheduled for deletion.
check_for_deletes($anvil);
# Check for new files
check_incoming($anvil);
# Check for files we should have but don't yet have.
find_missing_files($anvil);
}
# We're done
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Private functions. #
#############################################################################################################
# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't
# exist on this machine. For those entries, we will search for the file on other machines. The one exception
# is server definition files. For those, we write the file out from the server's 'definition' table entry.
#
# When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied
# to this host. Only machines on the same subnet are searched. The search pattern is;
#
# Nodes; 1. Check for the file on the peer.
# 2. Check for the file on Strikers, in alphabetical order.
# 3. Check for the file on DR host, if available.
# 4. Check other nodes, in alphabetical order.
# 5. Check other DR hosts, in alphabetical order.
# Striker; 1. Check for the file on other Strikers, in alphabetical order.
# 2. Check for the file on DR hosts, if available
# 3. Check for the file on Anvil! nodes.
# DR Host; 1. Check for the file on Strikers, in alphabetical order.
# 2. Check for the file on Anvil! nodes.
# * If a file can't be found, it will try again every so often until it is found.
# * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the
# md5sum matches. At this point, it is moved into the proper directory.
sub find_missing_files
{
my ($anvil) = @_;
# What am I? This will impact how missing files are found.
my $query = "
SELECT
a.file_uuid,
a.file_directory,
a.file_name,
a.file_size,
a.file_md5sum
FROM
files a,
hosts b,
file_locations c
WHERE
b.host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
AND
a.file_uuid = c.file_location_file_uuid
AND
b.host_uuid = c.file_location_host_uuid
ORDER BY
a.file_directory ASC,
a.file_name ASC,
b.host_name ASC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_uuid = $row->[0];
my $file_directory = $row->[1];
my $file_name = $row->[2];
my $file_size = $row->[3];
my $file_md5sum = $row->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:file_uuid' => $file_uuid,
's2:file_directory' => $file_directory,
's3:file_name' => $file_name,
}});
my $test_file = $file_directory."/".$file_name;
$test_file =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_file => $test_file }});
if (not -e $test_file)
{
# Missing file!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $test_file }});
# Find what target, if any, we'll the file from.
my ($found) = find_file($anvil, $file_uuid, $test_file, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
}
}
### TODO: Left off here.
return(0);
}
### TODO: Add sorting by preferred target
# This looks for a file on another system. The exact order of search depends on what kind of machine we are.
sub find_file
{
my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_;
my $found = 0;
# What are my IPs?
my $local_host = $anvil->Get->short_host_name();
$anvil->Network->get_ips();
foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
{
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
my $ip = $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
my $subnet_mask = $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
my $network = $anvil->Network->get_network({ip => $ip, subnet_mask => $subnet_mask});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:interface' => $interface,
's2:ip' => $ip,
's3:subnet_mask' => $subnet_mask,
's4:network' => $network,
}});
my $type = "other";
my $sort = $interface;
if ($interface =~ /^((?:bc|s|if)n)(\d+)/)
{
$type = $1;
$sort = $2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:type' => $type,
's2:sort' => $sort,
}});
}
$anvil->data->{local_ip}{by_network}{$network}{type} = $type;
$anvil->data->{local_ip}{by_network}{$network}{'sort'} = $sort;
$anvil->data->{local_ip}{by_network}{$network}{interface} = $interface;
$anvil->data->{local_ip}{by_network}{$network}{ip} = $ip;
$anvil->data->{local_ip}{by_network}{$network}{subnet_mask} = $subnet_mask;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:local_ip::by_network::${network}::type" => $anvil->data->{local_ip}{by_network}{$network}{type},
"s1:local_ip::by_network::${network}::sort" => $anvil->data->{local_ip}{by_network}{$network}{'sort'},
"s2:local_ip::by_network::${network}::interface" => $anvil->data->{local_ip}{by_network}{$network}{interface},
"s3:local_ip::by_network::${network}::ip" => $anvil->data->{local_ip}{by_network}{$network}{ip},
"s4:local_ip::by_network::${network}::subnet_mask" => $anvil->data->{local_ip}{by_network}{$network}{subnet_mask},
}});
}
# Create a hash we can sort through that are on the same subnet_mask as us.
my $query = "
SELECT
a.host_uuid,
a.host_name,
a.host_type,
b.file_directory,
b.file_name,
b.file_size,
b.file_md5sum
FROM
hosts a,
files b,
file_locations c
WHERE
a.host_uuid = c.file_location_host_uuid
AND
b.file_uuid = c.file_location_file_uuid
AND
b.file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
a.host_uuid != ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
ORDER BY
file_mtime DESC;
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $host_uuid = $row->[0];
my $host_name = $row->[1];
my $host_type = $row->[2];
my $file_directory = $row->[3];
my $file_name = $row->[4];
my $file_size = $row->[5];
my $file_md5sum = $row->[6];
my $test_file = $file_directory."/".$file_name;
$test_file =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_uuid => $host_uuid,
host_name => $host_name,
host_type => $host_type,
file_directory => $file_directory,
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
test_file => $test_file,
}});
# What IP addresses are on this machine?
my $query = "
SELECT
ip_address_address,
ip_address_subnet_mask
FROM
ip_addresses
WHERE
ip_address_note != 'DELETED'
AND
ip_address_host_uuid = ".$anvil->Database->quote($host_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $ip_address_address = $row->[0];
my $ip_address_subnet_mask = $row->[1];
my $network = $anvil->Network->get_network({ip => $ip_address_address, subnet_mask => $ip_address_subnet_mask});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
ip_address_address => $ip_address_address,
ip_address_subnet_mask => $ip_address_subnet_mask,
network => $network,
}});
# Are we on the same subnet?
if (exists $anvil->data->{local_ip}{by_network}{$network})
{
# We're on the same subnet!
my $type = $anvil->data->{local_ip}{by_network}{$network}{type};
my $sort = $anvil->data->{local_ip}{by_network}{$network}{'sort'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:type" => $type,
"s2:sort" => $sort,
}});
# Record.
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip} = $ip_address_address;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name} = $host_name;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid} = $host_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:peer_ip::${host_type}::${type}::${sort}::name" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name},
"s2:peer_ip::${host_type}::${type}::${sort}::ip" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip},
"s3:peer_ip::${host_type}::${type}::${sort}::host_uuid" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid},
}});
}
}
}
### TODO: Do this according to what we are.
# Sort through what we've found.
my $file_found = 0;
my $searched_hosts = {};
foreach my $host_type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}})
{
last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
foreach my $type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}})
{
last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }});
foreach my $sort (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}{$type}})
{
last if $file_found;
my $ip = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip};
my $name = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name};
my $host_uuid = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid};
my $remote_user = "admin";
my $password = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:host_type' => $host_type,
's2:type' => $type,
's3:sort' => $sort,
's4:name' => $name,
's5:ip' => $ip,
's6:host_uuid' => $host_uuid,
's7:remote_user' => $remote_user,
's8:password' => $password,
}});
if ((exists $searched_hosts->{$name}) && ($searched_hosts->{$name}))
{
# Already searched this host, this is just a different IP. Skip it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0282", variables => {
host_name => $name,
ip => $ip,
}});
next;
}
$searched_hosts->{$name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "searched_hosts->{$name}" => $searched_hosts->{$name} }});
### NOTE: There's a bug in Net::SSH2 on RHEL8 where passwordless SSH doesn't
### work. So for now, we'll manually pull in passwords from the
### anvil.conf using 'hosts::<host_name>::password' or
### 'hosts::<host_uuid>::password'. This will be removed when the bug
### is fixed.
if ((exists $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}) && ($anvil->data->{hosts}{$host_uuid}{password}{$remote_user}))
{
$password = $anvil->data->{hosts}{$host_uuid}{password}{$remote_user};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
}
elsif ((exists $anvil->data->{hosts}{$name}{password}{$remote_user}) && ($anvil->data->{hosts}{$name}{password}{$remote_user}))
{
$password = $anvil->data->{hosts}{$host_uuid}{password};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
}
### NOTE: This might take a while for the call to return if we're md5sum'ing
### a large file like an ISO.
### TODO: Should we ask for the size, then follow up with the md5sum for
### files that are over a certain size? Or will this get called rarely
### enough in practice that it doesn't really matter?
# If the file is found, we'll parse these out.
my $remote_size = 0;
my $remote_md5sum = "";
my ($output, $error, $return_code) = $anvil->Remote->call({
shell_call => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum".$anvil->Log->switches,
remote_user => $remote_user,
password => $password,
target => $ip,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
foreach my $line (split/\n/, $output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
if ($line =~ /^size: \[(\d+)\]$/)
{
$remote_size = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_size => $remote_size }});
}
if ($line =~ /^md5sum: \[(.*)\]$/)
{
$remote_md5sum = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_md5sum => $remote_md5sum }});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:remote_size' => $remote_size,
's2:file_size' => $file_size,
's3:remote_md5sum' => $remote_md5sum,
's4:file_md5sum' => $file_md5sum,
}});
### Do I really need to match sizes if the md5sum is the same?
if (($remote_size eq $file_size) && ($remote_md5sum eq $file_md5sum))
{
# Pull it over!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
}});
my $failed = $anvil->Storage->rsync({
debug => 2,
destination => $full_path,
password => $password,
source => $remote_user."\@".$ip.":".$full_path,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
if (-f $full_path)
{
# Got it!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }});
# Verify the md5sum.
my $local_md5sum = $anvil->Get->md5sum({file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
local_md5sum => $local_md5sum,
file_md5sum => $file_md5sum,
}});
if ($local_md5sum eq $file_md5sum)
{
# Success!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }});
$file_found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
last;
}
else
{
# Failed. :(
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }});
unlink $full_path;
}
}
else
{
# Failed to rsync.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
}});
}
}
else
{
# Doesn't match what we're looking for.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
remote_md5sum => $remote_md5sum,
file_md5sum => $file_md5sum,
file_size => $file_size,
say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
remote_size => $remote_size,
say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}),
}});
}
}
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
return($found);
}
# This handles deleting a file.
sub check_incoming
{
my ($anvil) = @_;
# This hash shouldn't exist, but lets be safe...
if (exists $anvil->data->{scan}{directories})
{
delete $anvil->data->{scan}{directories};
}
# Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({
debug => 3,
directory => $anvil->data->{path}{directories}{shared}{base},
recursive => 1,
});
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"});
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
{
# Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc)
next if $full_path =~ /^\/mnt\/shared\/temp\//;
# Skip if this isn't a file.
my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path,
file_type => $file_type,
}});
next if $file_type ne "file";
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_name => $file_name,
file_size => $file_size,
file_mtime => $file_mtime,
file_mimetype => $file_mimetype,
file_executable => $file_executable,
say_mimetype => $say_mimetype,
}});
# Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
my ($file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = get_file_db_info($anvil, "", $file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_uuid => $file_uuid,
recorded_size => $recorded_size,
recorded_mtime => $recorded_mtime,
recorded_md5sum => $recorded_md5sum,
}});
# Calculate the md5sum?
my $file_md5sum = $recorded_md5sum;
if ((not $file_uuid) or ($file_size != $recorded_size))
{
# Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer
# version down from elsewhere.
if (($file_uuid) && ($file_mtime <= $recorded_mtime))
{
# We've got an older file, we need to update.
pull_file($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum);
# TODO: Now see if it exists and, if it does, re-stat it. If not, loop to the
# next file and skip this one.
}
# Now generate the md5sum. If this file is over 128 MiB, warn the user that it might
# take a while.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }});
if ($file_size > (128 * (2 ** 20)))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => {
size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
}});
}
# Update (or get) the md5sum.
$file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
say_mimetype => $say_mimetype,
file_md5sum => $file_md5sum,
}});
# Insert or update the files entry.
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
}
# If we still don't have a file UUID for some reason, skip this file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
next if not $file_uuid;
# Make sure we know about this file on this system
my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => 3,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $anvil->data->{sys}{host_uuid},
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_locatiom_uuid => $file_locatiom_uuid }});
# Are we in the incoming directory? If so, move the file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path,
incoming_directory => $incoming_directory,
}});
if ($full_path =~ /^$incoming_directory/)
{
# If it's a definition file, we'll move it to
# 'path::directories::shared::definitions', otherwise we'll move it to
# 'path::directories::shared::files'.
my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
$target .= "/";
$target =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { target => $target }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => {
file => $full_path,
target => $target,
}});
$anvil->Storage->move_file({
debug => 3,
source_file => $full_path,
target_file => $target,
});
# Update the file_directory.
my $say_directory =~ s/\/$//;
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $say_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
}
}
return(0);
}
# This method finds a file elsewhere on the network and pulls it to here.
sub pull_file
{
my ($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = @_;
$file_uuid = "" if not defined $file_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
recorded_size => $recorded_size,
recorded_mtime => $recorded_mtime,
recorded_md5sum => $recorded_md5sum,
}});
# Find the hosts with this file, then connect to it to see if the size is the same as what we want.
# If so, pull it down...
### TODO
return(0);
}
# This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is
# returned. If it is found, the file size as recorded in the database is returned.
sub get_file_db_info
{
my ($anvil, $file_md5sum, $file_name) = @_;
$file_md5sum = "" if not defined $file_md5sum;
$file_name = "" if not defined $file_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_md5sum => $file_md5sum,
file_name => $file_name,
}});
# Get the file size and file uuid, if possible.
# If I have the md5sum, search using that. If I have the filename only, then we'll fall back to that.
my $query = "
SELECT
file_uuid,
file_size,
file_mtime,
file_md5sum
FROM
files
WHERE
";
if ($file_md5sum)
{
$query .= " file_md5sum = ".$anvil->Database->quote($file_md5sum)."\n";
}
elsif ($file_name)
{
$query .= " file_name = ".$anvil->Database->quote($file_name)."\n";
}
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
if (not $count)
{
# File wasn't found in the database
return("", 0, 0, "");
}
my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : 0;
my $file_mtime = defined $results->[0]->[2] ? $results->[0]->[2] : 0;
$file_md5sum = defined $results->[0]->[3] ? $results->[0]->[3] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_uuid => $file_uuid,
file_size => $file_size,
file_mtime => $file_mtime,
file_md5sum => $file_md5sum,
}});
return($file_uuid, $file_size, $file_mtime, $file_md5sum);
}
# This handles toggling a file to marked or unmarked as a script.
sub handle_script
{
my ($anvil) = @_;
if (not $anvil->data->{switches}{file})
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"});
$anvil->Job->update_progress({
progress => 0,
message => "error_0055",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 3});
}
# Find the file_uuid (we don't actually care about the file size, mtime or md5sum).
my ($file_uuid, $file_size, $recorded_mtime, $file_md5sum) = get_file_db_info($anvil, "", $anvil->data->{switches}{file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
# Toggle the executable bits.
my $executable = 0;
if ($anvil->data->{switches}{'is-script'})
{
# Is it already executable?
if (-x $anvil->data->{switches}{file})
{
# Switch it on
$executable = 1;
$anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a+x"});
}
else
{
# Already a script.
}
}
else
{
# Is it executable?
if (-x $anvil->data->{switches}{file})
{
# Switch it off.
$executable = 1;
$anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a-x"});
}
else
{
# Already not a script.
}
}
# If we have a file UUID, update the 'file_type
if ($file_uuid)
{
# Load the details.
my $query = "
SELECT
file_name,
file_directory,
file_size,
file_md5sum,
file_type,
file_mtime
FROM
files
WHERE
file_uuid = ".$anvil->Database->quote($file_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
my $file_name = $results->[0]->[0];
my $file_directory = $results->[0]->[1];
my $file_size = $results->[0]->[2];
my $file_md5sum = $results->[0]->[3];
my $file_type = $results->[0]->[4];
my $file_mtime = $results->[0]->[5];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
}});
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
{
# Figure out what the file type is and update.
my $mimetype = mimetype($file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }});
my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
}
elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
{
# Change the file tpye to "script"
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => "script",
});
}
}
return(0);
}
# This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about.
sub convert_mimetype
{
my ($anvil, $mimetype, $file, $executable) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
mimetype => $mimetype,
file => $file,
executable => $executable,
}});
my $say_mimetype = "other";
if ($mimetype)
{
if ($mimetype =~ /cd-image/)
{
$say_mimetype = "iso";
}
elsif ($mimetype =~ /xml/)
{
# This might be a definition, but look inside it to be sure.
my ($is_domain, $return_code) = $anvil->System->call({debug => 3, shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '</domain>' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi" });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { is_domain => $is_domain, return_code => $return_code }});
if ($is_domain)
{
$say_mimetype = "definition";
}
}
elsif ($mimetype =~ /rpm$/)
{
$say_mimetype = "rpm";
}
elsif ($mimetype =~ /disk-image/)
{
$say_mimetype = "disk-image";
}
elsif ($executable)
{
$say_mimetype = "script";
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_mimetype => $say_mimetype }});
return($say_mimetype);
}
# This looks for any files with the file_type of 'DELETED' and that has an entry in file_locations for this
# machine. Any that are found will have their file deleted and the file_locations entry deleted.
sub check_for_deletes
{
my ($anvil) = @_;
my $query = "
SELECT
a.file_uuid,
b.file_location_uuid,
a.file_directory || '/' || a.file_name AS full_path
FROM
files a,
file_locations b
WHERE
a.file_uuid = b.file_location_file_uuid
AND
a.file_type = 'DELETED'
AND
b.file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_uuid = $row->[0];
my $file_location_uuid = $row->[1];
my $full_path = $row->[2];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:file_uuid' => $file_uuid,
's2:file_location_uuid' => $file_location_uuid,
's3:full_path' => $full_path,
}});
# Get rid of it (if it actually exists) and then remove it from file_locations.
if (-e $full_path)
{
# Delete it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }});
unlink $full_path;
# Sleep and verify
sleep 1;
if (-e $full_path)
{
# Failed to delete...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }});
return("");
}
else
{
# Deleted successfully.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
}
}
# Delete the entry from file_locations, if needed.
$query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
return(0);
}
# This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the
# database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a
# file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts.
sub handle_delete
{
my ($anvil) = @_;
my $full_path = $anvil->data->{switches}{file};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }});
if (not $full_path)
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
$anvil->Job->update_progress({
progress => 0,
message => "error_0052",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 3});
}
elsif ($full_path !~ /^\/mnt\/shared\//)
{
# We don't do that here...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0053,!!file!".$full_path."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 6});
}
# What's the UUID of this file? I collect all the data for the update, if appropriate.
my $file_name = ($full_path =~ /^.*\/(.*)$/)[0];
my $file_directory = ($full_path =~ /^(.*?)\/$file_name$/)[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
full_name => $file_name,
file_directory => $file_directory,
}});
my $query = "
SELECT
file_uuid,
file_size,
file_md5sum,
file_type,
file_mtime
FROM
files
WHERE
file_name = ".$anvil->Database->quote($file_name)."
AND
file_directory = ".$anvil->Database->quote($file_directory)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : "";
my $file_md5sum = defined $results->[0]->[2] ? $results->[0]->[2] : "";
my $file_type = defined $results->[0]->[3] ? $results->[0]->[3] : "";
my $file_mtime = defined $results->[0]->[4] ? $results->[0]->[4] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
}});
# If I have the file_uuid, and we've been asked to delete it everywhere, mark it as DELETEd in the
# database. This way, if anything goes wrong below, future runs will try (again) to delete the file
# itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt
# to try again.
if (($file_uuid) && ($anvil->data->{switches}{everywhere}))
{
# Yup.
($file_uuid) = $anvil->Database->insert_or_update_files({
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => "DELETED",
file_mtime => $file_mtime,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
}
# Does the file exist?
if (-e $full_path)
{
# Delete it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0263", variables => { file => $full_path }});
unlink $full_path;
# Sleep and verify
sleep 1;
if (-e $full_path)
{
# Failed to delete...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284"});
return("");
}
else
{
# Deleted successfully.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
}
}
# Now, if I have a file_uuid, delete it from file_locations if it exists.
if ($file_uuid)
{
# Delete the entry from file_locations, if needed.
my $query = "
DELETE FROM
file_locations
WHERE
file_location_file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
return(0);
}
# This handles renaming files.
sub handle_rename
{
my ($anvil) = @_;
# Do we have the current file name and the new one?
if (not $anvil->data->{switches}{file})
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0049"});
$anvil->Job->update_progress({
progress => 0,
message => "error_0049",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 3});
}
elsif (not $anvil->data->{switches}{to})
{
# We need a target
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0050", variables => { file => $anvil->data->{switches}{file} }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0050,!!file!".$anvil->data->{switches}{file}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 4});
}
elsif (not -f $anvil->data->{switches}{file})
{
# The file to rename doesn't exist.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0051", variables => { file => $anvil->data->{switches}{file} }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0051,!!file!".$anvil->data->{switches}{file}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 5});
}
elsif (-e $anvil->data->{switches}{to})
{
# There's already a file (or directory or something) with that name.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
$anvil->Job->update_progress({
progress => 0,
message => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
$anvil->nice_exit({exit_code => 6});
}
### TODO: Left off here
return(0);
}
Reference in New Issue View Git Blame Copy Permalink