@ -131,7 +131,7 @@ else
check_incoming($anvil); check_incoming($anvil);
# Check for files we should have but don't yet have. # Check for files we should have but don't yet have.
find_missing_files($anvil); # find_missing_files($anvil);
} }
# We're done # We're done
@ -167,33 +167,33 @@ sub find_missing_files
my ($anvil) = @_; my ($anvil) = @_;
# What am I? This will impact how missing files are found. # What am I? This will impact how missing files are found.
my $query = " # my $query = "
SELECT # SELECT
file_location_file_uuid # file_location_file_uuid
FROM # FROM
file_locations # file_locations
WHERE # WHERE
file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})." # file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})."
;"; # ;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug , list => { query => $query }});# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2 , list => { query => $query }});
#
my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $ THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});# my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};# my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug , list => { # $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2 , list => {
results => $results, # results => $results,
count => $count, # count => $count,
}});# }});
foreach my $row (@{$results})# foreach my $row (@{$results})
{# {
my $file_location_file_uuid = $row->[0];# my $file_location_file_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug , list => { # $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2 , list => {
file_location_file_uuid => $file_location_file_uuid, # file_location_file_uuid => $file_location_file_uuid,
}});# }});
#
### TODO: How to handle when the file with the same name exists on 2+ machines with # ### TODO: How to handle when the file with the same name exists on 2+ machines with
### different md5sums. Use the most recent mtime?# ### different md5sums. Use the most recent mtime?
# Read in the file details.# # Read in the file details.
}# }
# Read in any entries from 'file_locations'. # Read in any entries from 'file_locations'.
@ -212,31 +212,64 @@ sub check_incoming
} }
# Read any files in '/mnt/shared/incoming'. # Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({ $anvil->Storage->scan_directory({
debug => 3 , debug => 2 ,
directory => $anvil->data->{path}{directories}{shared}{incoming }, directory => $anvil->data->{path}{directories}{shared}{base },
recursive => 0 , recursive => 1 ,
}); });
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"});
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}}) foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
{ {
# Is this a file? # Skip if this isn't a file.
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_type = $anvil->data->{scan}{directories}{$full_path}{type}; my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
full_path => $full_path,
file_type => $file_type,
}});
next if $file_type ne "file";
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size}; my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0; my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
full_path => $full_path,
file_name => $file_name, file_name => $file_name,
file_type => $file_type,
file_size => $file_size, file_size => $file_size,
file_mtime => $file_mtime,
file_mimetype => $file_mimetype, file_mimetype => $file_mimetype,
file_executable => $file_executable, file_executable => $file_executable,
say_mimetype => $say_mimetype,
}}); }});
next if $file_type ne "file";
# If this file is over 128 MiB, warn the user that it might take a second # Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
my ($file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = get_file_db_info($anvil, "", $file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
recorded_size => $recorded_size,
recorded_mtime => $recorded_mtime,
recorded_md5sum => $recorded_md5sum,
}});
# Calculate the md5sum?
my $file_md5sum = $recorded_md5sum;
if ((not $file_uuid) or ($file_size != $recorded_size))
{
# Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer
# version down from elsewhere.
if (($file_uuid) && ($file_mtime <= $recorded_mtime))
{
# We've got an older file, we need to update.
pull_file($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum);
# TODO: Now see if it exists and, if it does, re-stat it. If not, loop to the
# next file and skip this one.
}
# Now generate the md5sum. If this file is over 128 MiB, warn the user that it might
# take a while.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }});
if ($file_size > (128 * (2 ** 20))) if ($file_size > (128 * (2 ** 20)))
{ {
@ -244,24 +277,30 @@ sub check_incoming
size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}), size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
}}); }});
} }
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
my $file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path}); # Update (or get) the md5sum.
$file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
say_mimetype => $say_mimetype, say_mimetype => $say_mimetype,
file_md5sum => $file_md5sum, file_md5sum => $file_md5sum,
}}); }});
# Do we know about this file? If not, file_uuid will be blank when we call the # Insert or update the files entry.
# insert_or_update. If we do, it will update the file name, if needed. ($file_uuid) = $anvil->Database->insert_or_update_files({
my ($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2, debug => 2,
file_uuid => get_file_uuid($anvil, $file_md5sum, $file_name) , file_uuid => $file_uuid ,
file_name => $file_name, file_name => $file_name,
file_size => $file_size, file_size => $file_size,
file_md5sum => $file_md5sum, file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype, file_type => $say_mimetype,
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
}
# If we still don't have a file UUID for some reason, skip this file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
next if not $file_uuid;
# Make sure we know about this file on this system # Make sure we know about this file on this system
my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({ my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
@ -271,7 +310,14 @@ sub check_incoming
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_locatiom_uuid => $file_locatiom_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_locatiom_uuid => $file_locatiom_uuid }});
# Not move it. If it's a definition file, we'll move it to # Are we in the incoming directory? If so, move the file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
full_path => $full_path,
incoming_directory => $incoming_directory,
}});
if ($full_path =~ /^$incoming_directory/)
{
# If it's a definition file, we'll move it to
# 'path::directories::shared::definitions', otherwise we'll move it to # 'path::directories::shared::definitions', otherwise we'll move it to
# 'path::directories::shared::files'. # 'path::directories::shared::files'.
my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files}; my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
@ -289,13 +335,33 @@ sub check_incoming
target_file => $target, target_file => $target,
}); });
} }
}
return(0);
}
# This method finds a file elsewhere on the network and pulls it to here.
sub pull_file
{
my ($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = @_;
$file_uuid = "" if not defined $file_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
recorded_size => $recorded_size,
recorded_mtime => $recorded_mtime,
recorded_md5sum => $recorded_md5sum,
}});
# Find the hosts with this file, then connect to it to see if the size is the same as what we want.
# If so, pull it down...
### TODO
return(0); return(0);
} }
# This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is # This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is
# returned. # returned. If it is found, the file size as recorded in the database is returned.
sub get_file_uuid sub get_file_db_info
{ {
my ($anvil, $file_md5sum, $file_name) = @_; my ($anvil, $file_md5sum, $file_name) = @_;
$file_md5sum = "" if not defined $file_md5sum; $file_md5sum = "" if not defined $file_md5sum;
@ -305,12 +371,15 @@ sub get_file_uuid
file_name => $file_name, file_name => $file_name,
}}); }});
### TODO: At some point, we'll need to deal with the possibility that the same file name with # Get the file size and file uuid, if possible.
### different md5sums might exist in the database.
# If I have the md5sum, search using that. If I have the filename only, then we'll fall back to that. # If I have the md5sum, search using that. If I have the filename only, then we'll fall back to that.
my $query = " my $query = "
SELECT SELECT
file_uuid file_uuid,
file_size,
file_mtime,
file_md5sum
FROM FROM
files files
WHERE WHERE
@ -332,14 +401,20 @@ WHERE
if (not $count) if (not $count)
{ {
# File wasn't found in the database # File wasn't found in the database
return(""); return("", 0, 0, "" );
} }
my $file_uuid = $results->[0]->[0]; my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : "";
my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : 0;
my $file_mtime = defined $results->[0]->[2] ? $results->[0]->[2] : 0;
$file_md5sum = defined $results->[0]->[3] ? $results->[0]->[3] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_size => $file_size,
file_mtime => $file_mtime,
file_md5sum => $file_md5sum,
}}); }});
return($file_uuid); return($file_uuid, $file_size, $file_mtime, $file_md5sum );
} }
# This handles toggling a file to marked or unmarked as a script. # This handles toggling a file to marked or unmarked as a script.
@ -359,8 +434,8 @@ sub handle_script
$anvil->nice_exit({exit_code => 3}); $anvil->nice_exit({exit_code => 3});
} }
# Find the file_uuid. # Find the file_uuid (we don't actually care about the file size, mtime or md5sum) .
my ($file_uuid) = get_file_uuid ($anvil, "", $anvil->data->{switches}{file}); my ($file_uuid, $file_size, $recorded_mtime, $file_md5sum) = get_file_db_info ($anvil, "", $anvil->data->{switches}{file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
# Toggle the executable bits. # Toggle the executable bits.
@ -403,7 +478,8 @@ SELECT
file_name, file_name,
file_size, file_size,
file_md5sum, file_md5sum,
file_type file_type,
file_mtime
FROM FROM
files files
WHERE WHERE
@ -417,11 +493,13 @@ WHERE
my $file_size = $results->[0]->[1]; my $file_size = $results->[0]->[1];
my $file_md5sum = $results->[0]->[2]; my $file_md5sum = $results->[0]->[2];
my $file_type = $results->[0]->[3]; my $file_type = $results->[0]->[3];
my $file_mtime = $results->[0]->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_name => $file_name, file_name => $file_name,
file_size => $file_size, file_size => $file_size,
file_md5sum => $file_md5sum, file_md5sum => $file_md5sum,
file_type => $file_type, file_type => $file_type,
file_mtime => $file_mtime,
}}); }});
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'})) if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
{ {
@ -429,23 +507,15 @@ WHERE
my $mimetype = mimetype($file_name); my $mimetype = mimetype($file_name);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }});
my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, $executable ); my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0 );
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
$anvil->Database->insert_or_update_files({
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $say_mimetype,
});
# Change the file tpye to "say_mimetype".
$anvil->Database->insert_or_update_files({ $anvil->Database->insert_or_update_files({
debug => 2, debug => 2,
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file}, file_name => $anvil->data->{switches}{file},
file_size => $file_size, file_size => $file_size,
file_md5sum => $file_md5sum, file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype, file_type => $say_mimetype,
}); });
} }
@ -458,6 +528,7 @@ WHERE
file_name => $anvil->data->{switches}{file}, file_name => $anvil->data->{switches}{file},
file_size => $file_size, file_size => $file_size,
file_md5sum => $file_md5sum, file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => "script", file_type => "script",
}); });
} }
Digimer
6 years ago