seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* There were still references in anvil-manage-files to 'file_locations' -> 'file_location_host_uuid'. Had to rework some logic to get things working. More testing needed, but so far at least the "missing file" function is working again.

Browse Source 
* Added missing always-available switchs in Get->switches
* Create Storage->_wait_if_changing() to check to see if a file's size is changing and, if so, not return until it stops.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 2 years ago
parent e2a42870e5
commit be84a23924
5 changed files with 394 additions and 419 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      Anvil/Tools/Get.pm
  2. 102
      Anvil/Tools/Storage.pm
  3. 38
      man/anvil-manage-files.8
  4. 5
      share/words.xml
  5. 660
      tools/anvil-manage-files

4
Anvil/Tools/Get.pm
Unescape View File

@ -2297,12 +2297,14 @@ sub switches
next if $set_switch eq "?";
next if $set_switch eq "h";
next if $set_switch eq "help";
next if $set_switch eq "log-secure";
next if $set_switch eq "log-db-transactions";
next if $set_switch eq "raw";
next if $set_switch eq "resync-db";
next if $set_switch eq "v";
next if $set_switch eq "vv";
next if $set_switch eq "vvv";
next if $set_switch eq "vvvv";
next if $set_switch eq "log-secure";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set_switch => $set_switch }});
my $found = 0;

102
Anvil/Tools/Storage.pm
Unescape View File

@ -46,6 +46,7 @@ my $THIS_FILE = "Storage.pm";
# update_file
# write_file
# _create_rsync_wrapper
# _wait_if_changing
=pod
@ -5151,6 +5152,7 @@ fi";
# Private functions #
#############################################################################################################
=head2
This does the actual work of creating the C<< expect >> wrapper script and returns the path to that wrapper for C<< rsync >> calls.
@ -5227,4 +5229,104 @@ expect eof
return($wrapper_script);
}
=head3 _wait_if_changing
This takes a full path to a file, and watches it for at specified number of seconds to see if the size is changing. If it is, this method waits until the file size stops changing.
Parameters;
=head3 file (required)
This is the full path to the file. If the file is not found, C<< !!error!! >> is returned.
=head3 delay (optional, default '2')
This is how long to wait before checking to see if the file has changed.
=head3 last_size (optional)
If this is set, it's the first size we compare against. If not passed, the size will be checked.
=cut
sub _wait_if_changing
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->_create_rsync_wrapper()" }});
# Check my parameters.
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $delay = defined $parameter->{delay} ? $parameter->{delay} : "";
my $last_size = defined $parameter->{last_size} ? $parameter->{last_size} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file => $file,
delay => $delay,
last_size => $last_size,
}});
if (not $delay)
{
$delay = 2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }});
}
elsif (($delay =~ /\D/) or ($delay == 0))
{
$delay = 2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }});
}
if (not -e $file)
{
return("!!error!!");
}
if (not $last_size)
{
$last_size = (stat($file))[7];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")",
}});
}
my $waiting = 1;
while ($waiting)
{
sleep $delay;
my $new_size = (stat($file))[7];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file => $file,
last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")",
}});
if ($new_size == $last_size)
{
# Size seems stable
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
}
else
{
# Might still be updating, wait.
my $difference = $new_size - $last_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0724", variables => {
file => $file,
old_size_bytes => $anvil->Convert->add_commas({number => $last_size}),
old_size_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}),
new_size_bytes => $anvil->Convert->add_commas({number => $new_size}),
new_size_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $new_size}),
difference_bytes => $anvil->Convert->add_commas({number => $difference}),
difference_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $difference}),
}});
$last_size = $new_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { last_size => $last_size }});
}
}
return(0);
}
1;

38
man/anvil-manage-files.8
Unescape View File

@ -0,0 +1,38 @@
.\" Manpage for the Anvil! server removal tool
.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
.TH anvil-manage-files "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform"
.SH NAME
anvil-manage-files \- This program manages the files sync'ed across machines in the Anvil! cluster
.SH SYNOPSIS
.B anvil-manage-files
\fI\,<command> \/\fR[\fI\,options\/\fR]
.SH DESCRIPTION
This handles synchronizing files, typically ISO used to build new servers and scripts used to manage them, across Anvil! nodes.
.TP
.SH OPTIONS
.TP
\-?, \-h, \fB\-\-help\fR
Show this man page.
.TP
\fB\-\-log-secure\fR
When logging, record sensitive data, like passwords.
.TP
\-v, \-vv, \-vvv
Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
.SS "Commands:"
.TP
\fB\-\-delete\fR
This will delete the \fB\-\-file\fR </path/to/file> from the entire Anvil! cluster.
.TP
This action is permanent!
.TP
\fB\-\-job-uuid\fR <name>
The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
.IP
.SH AUTHOR
Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
.SH "REPORTING BUGS"
Report bugs to users@clusterlabs.org
", "download", "everywhere", "file", "is-script", "job-uuid", "rename", "to

5
share/words.xml
Unescape View File

@ -1664,8 +1664,8 @@ The fingerprint of: [#!variable!machine!#] has changed! Updating it's entry in k
<key name="log_0279">The md5sum of file: [#!variable!file!#] failed to match. Discarding the downloaded file.</key>
<key name="log_0280">Failed to download: [#!variable!file!#] from: [#!variable!host_name!# (#!variable!ip!#). Will look on other hosts (if any left).</key>
<key name="log_0281">The file: [#!variable!file!#] on: [#!variable!host_name!# (#!variable!ip!#]) doesn't match the file we're looking for.
- Wanted; md5sum: [#!variable!file_md5sum!#], size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)]
- Found; md5sum: [#!variable!remote_md5sum!#], size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)]
- Wanted; size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)]
- Found; size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)]
We will keep looking.</key>
<key name="log_0282">Already searched: [#!variable!host_name!# using another IP address, skipping this IP: [#!variable!ip!#].</key>
<key name="log_0283">Done.</key>
@ -2151,6 +2151,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
<key name="log_0721">The server: [#!variable!server!#] is ready to boot.</key>
<key name="log_0722">The server: [#!variable!server!#] was found to be running already, but it wasn't marked as booted. Marking it as if it just booted to handle any dependent servers.</key>
<key name="log_0723">The server: [#!variable!server!#] is configured to stay off, ignoring it.</key>
<key name="log_0724">The file: [#!variable!file!#] needs to be added to the database, but since the last scan it's size grew from: [#!variable!old_size_bytes!# (#!variables!old_size_hr!#)] to: [#!variable!new_size_bytes!# (#!variables!new_size_hr!#)]. A difference of: [#!variable!difference_bytes!# (#!variables!difference_hr!#)]. It might still be being uploaded, so we'll keep checking periodocally until the size stops changing.</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

660
tools/anvil-manage-files
Unescape View File

@ -13,10 +13,9 @@
# - If not found anywhere, remove it from 'file_locations' and send an alert.
# - If found, check the size. If it differs, recalculate the md5sum.
# - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If
# also used with '--everywhere', then all copies on all systems we know about will be deleted. This is
# done by registering a job against all known hosts. As such, if this is called and the target file
# doesn't exist, it just clears the job and then exits.
# - 4. If called with '--delete', remove from 'file_locations' and all copies on all systems. This is done by
# registering a job against all known hosts. As such, if this is called and the target file doesn't exist,
# it just clears the job and then exits.
# - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit.
#
# Exit codes;
@ -32,7 +31,7 @@
# -
#
# NOTE:
# -
# - remove unsyncs, add syncs.
#
use strict;
@ -52,25 +51,11 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new();
$anvil->data->{switches}{'delete'} = "";
$anvil->data->{switches}{download} = "";
$anvil->data->{switches}{everywhere} = "";
$anvil->data->{switches}{file} = "";
$anvil->data->{switches}{'is-script'} = "";
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'rename'} = "";
$anvil->data->{switches}{to} = "";
$anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"switches::delete" => $anvil->data->{switches}{'delete'},
"switches::download" => $anvil->data->{switches}{download},
"switches::everywhere" => $anvil->data->{switches}{everywhere},
"switches::file" => $anvil->data->{switches}{file},
"switches::is-script" => $anvil->data->{switches}{'is-script'},
"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'},
"switches::rename" => $anvil->data->{switches}{'rename'},
"switches::to" => $anvil->data->{switches}{to},
}});
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Get->switches({list => ["delete", "download", "file", "is-script", "job-uuid", "rename", "remove", "add", "to"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
# Connect or die
$anvil->Database->connect;
@ -171,59 +156,59 @@ sub find_missing_files
# What am I? This will impact how missing files are found.
my $query = "
SELECT
a.file_uuid,
a.file_directory,
a.file_name,
a.file_size,
a.file_md5sum
file_uuid,
file_directory,
file_name,
file_size,
file_md5sum
FROM
files a,
hosts b,
file_locations c
files
WHERE
b.host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
AND
a.file_uuid = c.file_location_file_uuid
AND
b.host_uuid = c.file_location_host_uuid
file_type != 'DELETED'
ORDER BY
a.file_directory ASC,
a.file_name ASC,
b.host_name ASC
file_directory ASC,
file_name ASC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_uuid = $row->[0];
my $file_directory = $row->[1];
my $file_name = $row->[2];
my $file_size = $row->[3];
my $file_md5sum = $row->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
my $file_uuid = $row->[0];
my $file_directory = $row->[1];
my $file_name = $row->[2];
my $file_size = $row->[3];
my $file_md5sum = $row->[4];
my $full_path = $file_directory."/".$file_name;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:file_uuid' => $file_uuid,
's2:file_directory' => $file_directory,
's3:file_name' => $file_name,
's4:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
's5:file_md5sum' => $file_md5sum,
's6:full_path' => $full_path,
}});
my $test_file = $file_directory."/".$file_name;
$test_file =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_file => $test_file }});
if (not -e $test_file)
# If we're a striker, we want all files.
my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type eq "striker")
{
# Missing file!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $test_file }});
if (not -e $full_path)
{
# Missing file!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $full_path }});
# Find what target, if any, we'll the file from.
my ($found) = find_file($anvil, $file_uuid, $test_file, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
# Find what target, if any, we'll the file from.
my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
}
}
}
@ -232,338 +217,167 @@ ORDER BY
return(0);
}
### TODO: Add sorting by preferred target
# This looks for a file on another system. The exact order of search depends on what kind of machine we are.
sub find_file
{
my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:file_uuid' => $file_uuid,
's2:full_path' => $full_path,
's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
's4:file_md5sum' => $file_md5sum,
}});
my $found = 0;
# What are my IPs?
my $local_host = $anvil->Get->short_host_name();
$anvil->Network->get_ips();
foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
# We want to search Striker's first, DR hosts second and nodes third.
$anvil->Database->get_hosts;
my $host_order = [];
foreach my $type ("striker", "dr", "node")
{
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
my $ip = $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
my $subnet_mask = $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask};
my $network = $anvil->Network->get_network({ip => $ip, subnet_mask => $subnet_mask});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:interface' => $interface,
's2:ip' => $ip,
's3:subnet_mask' => $subnet_mask,
's4:network' => $network,
}});
my $type = "other";
my $sort = $interface;
if ($interface =~ /^((?:bc|s|if)n)(\d+)/)
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type }});
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
{
$type = $1;
$sort = $2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:type' => $type,
's2:sort' => $sort,
my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name};
my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:host_name' => $host_name,
's2:host_type' => $host_type,
's3:host_uuid' => $host_uuid,
}});
next if $host_type ne $type;
next if $host_uuid eq $anvil->Get->host_uuid;
push @{$host_order}, $host_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
}
$anvil->data->{local_ip}{by_network}{$network}{type} = $type;
$anvil->data->{local_ip}{by_network}{$network}{'sort'} = $sort;
$anvil->data->{local_ip}{by_network}{$network}{interface} = $interface;
$anvil->data->{local_ip}{by_network}{$network}{ip} = $ip;
$anvil->data->{local_ip}{by_network}{$network}{subnet_mask} = $subnet_mask;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:local_ip::by_network::${network}::type" => $anvil->data->{local_ip}{by_network}{$network}{type},
"s1:local_ip::by_network::${network}::sort" => $anvil->data->{local_ip}{by_network}{$network}{'sort'},
"s2:local_ip::by_network::${network}::interface" => $anvil->data->{local_ip}{by_network}{$network}{interface},
"s3:local_ip::by_network::${network}::ip" => $anvil->data->{local_ip}{by_network}{$network}{ip},
"s4:local_ip::by_network::${network}::subnet_mask" => $anvil->data->{local_ip}{by_network}{$network}{subnet_mask},
}});
}
# Create a hash we can sort through that are on the same subnet_mask as us.
my $query = "
SELECT
a.host_uuid,
a.host_name,
a.host_type,
b.file_directory,
b.file_name,
b.file_size,
b.file_md5sum
FROM
hosts a,
files b,
file_locations c
WHERE
a.host_uuid = c.file_location_host_uuid
AND
b.file_uuid = c.file_location_file_uuid
AND
b.file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
a.host_uuid != ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
ORDER BY
file_mtime DESC;
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
# Now search.
my $file_found = 0;
foreach my $search_host_uuid (@{$host_order})
{
my $host_uuid = $row->[0];
my $host_name = $row->[1];
my $host_type = $row->[2];
my $file_directory = $row->[3];
my $file_name = $row->[4];
my $file_size = $row->[5];
my $file_md5sum = $row->[6];
my $test_file = $file_directory."/".$file_name;
$test_file =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_uuid => $host_uuid,
host_name => $host_name,
host_type => $host_type,
file_directory => $file_directory,
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
test_file => $test_file,
last if $file_found;
my $target_host = $anvil->data->{hosts}{host_uuid}{$search_host_uuid}{short_host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:search_host_uuid' => $search_host_uuid,
's2:target_host' => $target_host,
}});
# What IP addresses are on this machine?
my $query = "
SELECT
ip_address_address,
ip_address_subnet_mask
FROM
ip_addresses
WHERE
ip_address_note != 'DELETED'
AND
ip_address_host_uuid = ".$anvil->Database->quote($host_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
my $target_ip = $anvil->Network->find_target_ip({host_uuid => $search_host_uuid});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_ip => $target_ip }});
next if not $target_ip;
# See if the file is on the target and, if so, if it matches.
### NOTE: If we want to use md5sum again, use '--with-md5sum'
my $shell_call = $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path.$anvil->Log->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
# Test access
my $access = $anvil->Remote->test_access({target => $target_ip});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
next if not $access;
my $remote_size = 0;
my ($output, $error, $return_code) = $anvil->Remote->call({
shell_call => $shell_call,
target => $target_ip,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
foreach my $row (@{$results})
foreach my $line (split/\n/, $output)
{
my $ip_address_address = $row->[0];
my $ip_address_subnet_mask = $row->[1];
my $network = $anvil->Network->get_network({ip => $ip_address_address, subnet_mask => $ip_address_subnet_mask});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
ip_address_address => $ip_address_address,
ip_address_subnet_mask => $ip_address_subnet_mask,
network => $network,
}});
# Are we on the same subnet?
if (exists $anvil->data->{local_ip}{by_network}{$network})
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /^size: \[(\d+)\]$/)
{
# We're on the same subnet!
my $type = $anvil->data->{local_ip}{by_network}{$network}{type};
my $sort = $anvil->data->{local_ip}{by_network}{$network}{'sort'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:type" => $type,
"s2:sort" => $sort,
}});
# Record.
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip} = $ip_address_address;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name} = $host_name;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid} = $host_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:peer_ip::${host_type}::${type}::${sort}::name" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name},
"s2:peer_ip::${host_type}::${type}::${sort}::ip" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip},
"s3:peer_ip::${host_type}::${type}::${sort}::host_uuid" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid},
$remote_size = $1;
$found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
remote_size => $remote_size,
found => $found,
}});
}
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:found' => $found,
's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")",
's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
}});
next if not $found;
### TODO: Do this according to what we are.
# Sort through what we've found.
my $file_found = 0;
my $searched_hosts = {};
foreach my $host_type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}})
{
last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
foreach my $type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}})
if ($remote_size eq $file_size)
{
last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }});
foreach my $sort (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}{$type}})
{
last if $file_found;
my $ip = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip};
my $name = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name};
my $host_uuid = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid};
my $remote_user = "admin";
my $password = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:host_type' => $host_type,
's2:type' => $type,
's3:sort' => $sort,
's4:name' => $name,
's5:ip' => $ip,
's6:host_uuid' => $host_uuid,
's7:remote_user' => $remote_user,
's8:password' => $password,
}});
# Pull it over!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => {
file => $full_path,
host_name => $target_host,
ip => $target_ip,
}});
my $failed = $anvil->Storage->rsync({
debug => 2,
destination => $full_path,
source => "root\@".$target_ip.":".$full_path,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
if ((exists $searched_hosts->{$name}) && ($searched_hosts->{$name}))
{
# Already searched this host, this is just a different IP. Skip it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0282", variables => {
host_name => $name,
ip => $ip,
}});
next;
}
$searched_hosts->{$name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "searched_hosts->{$name}" => $searched_hosts->{$name} }});
### NOTE: There's a bug in Net::SSH2 on RHEL8 where passwordless SSH doesn't
### work. So for now, we'll manually pull in passwords from the
### anvil.conf using 'hosts::<host_name>::password' or
### 'hosts::<host_uuid>::password'. This will be removed when the bug
### is fixed.
if ((exists $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}) && ($anvil->data->{hosts}{$host_uuid}{password}{$remote_user}))
{
$password = $anvil->data->{hosts}{$host_uuid}{password}{$remote_user};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
}
elsif ((exists $anvil->data->{hosts}{$name}{password}{$remote_user}) && ($anvil->data->{hosts}{$name}{password}{$remote_user}))
{
$password = $anvil->data->{hosts}{$host_uuid}{password};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
}
if (-f $full_path)
{
# Got it!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }});
### NOTE: This might take a while for the call to return if we're md5sum'ing
### a large file like an ISO.
### TODO: Should we ask for the size, then follow up with the md5sum for
### files that are over a certain size? Or will this get called rarely
### enough in practice that it doesn't really matter?
# If the file is found, we'll parse these out.
my $remote_size = 0;
my $remote_md5sum = "";
my ($output, $error, $return_code) = $anvil->Remote->call({
shell_call => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum".$anvil->Log->switches,
remote_user => $remote_user,
password => $password,
target => $ip,
});
# Verify the md5sum.
my $local_md5sum = $anvil->Get->md5sum({file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
local_md5sum => $local_md5sum,
file_md5sum => $file_md5sum,
}});
foreach my $line (split/\n/, $output)
if ($local_md5sum eq $file_md5sum)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
if ($line =~ /^size: \[(\d+)\]$/)
{
$remote_size = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_size => $remote_size }});
}
if ($line =~ /^md5sum: \[(.*)\]$/)
{
$remote_md5sum = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_md5sum => $remote_md5sum }});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:remote_size' => $remote_size,
's2:file_size' => $file_size,
's3:remote_md5sum' => $remote_md5sum,
's4:file_md5sum' => $file_md5sum,
}});
# Success!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }});
### Do I really need to match sizes if the md5sum is the same?
if (($remote_size eq $file_size) && ($remote_md5sum eq $file_md5sum))
{
# Pull it over!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
}});
my $failed = $anvil->Storage->rsync({
debug => 2,
destination => $full_path,
password => $password,
source => $remote_user."\@".$ip.":".$full_path,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
if (-f $full_path)
{
# Got it!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }});
# Verify the md5sum.
my $local_md5sum = $anvil->Get->md5sum({file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
local_md5sum => $local_md5sum,
file_md5sum => $file_md5sum,
}});
if ($local_md5sum eq $file_md5sum)
{
# Success!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }});
$file_found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
last;
}
else
{
# Failed. :(
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }});
unlink $full_path;
}
}
else
{
# Failed to rsync.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
}});
}
$file_found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
last;
}
else
{
# Doesn't match what we're looking for.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => {
file => $full_path,
host_name => $name,
ip => $ip,
remote_md5sum => $remote_md5sum,
file_md5sum => $file_md5sum,
file_size => $file_size,
say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
remote_size => $remote_size,
say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}),
}});
# Failed. :(
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }});
unlink $full_path;
}
}
else
{
# Failed to rsync.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => {
file => $full_path,
host_name => $target_host,
ip => $target_ip,
}});
}
}
else
{
# Doesn't match what we're looking for.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => {
file => $full_path,
host_name => $target_host,
ip => $target_ip,
file_size => $file_size,
say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
remote_size => $remote_size,
say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}),
}});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
return($found);
return($file_found);
}
@ -577,6 +391,7 @@ sub check_incoming
{
delete $anvil->data->{scan}{directories};
}
# Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({
debug => 3,
@ -600,13 +415,15 @@ sub check_incoming
}});
next if $file_type ne "file";
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
my $full_path = $file_directory."/".$file_name;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_name => $file_name,
file_size => $file_size,
@ -614,6 +431,7 @@ sub check_incoming
file_mimetype => $file_mimetype,
file_executable => $file_executable,
say_mimetype => $say_mimetype,
full_path => $full_path,
}});
# Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
@ -626,11 +444,22 @@ sub check_incoming
}});
# Calculate the md5sum?
my $file_md5sum = $recorded_md5sum;
my $file_md5sum = $recorded_md5sum;
if ((not $file_uuid) or ($file_size != $recorded_size))
{
# Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer
# version down from elsewhere.
# It's possible the file is still uploading, so sleep for 2 seconds and see if the
# size is still changing.
my $last_size = $file_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")",
}});
$anvil->Storage->_wait_if_changing({
file => $full_path,
last_size => $file_size,
});
# Yes, caluclate md5sum, but first, do we have a size mismatch? If so, see if we need
# to pull a newer version down from elsewhere.
if (($file_uuid) && ($file_mtime <= $recorded_mtime))
{
# We've got an older file, we need to update.
@ -675,14 +504,6 @@ sub check_incoming
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
next if not $file_uuid;
# Make sure we know about this file on this system
my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => 3,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $anvil->data->{sys}{host_uuid},
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_locatiom_uuid => $file_locatiom_uuid }});
# Are we in the incoming directory? If so, move the file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path,
@ -694,14 +515,21 @@ sub check_incoming
# 'path::directories::shared::definitions', otherwise we'll move it to
# 'path::directories::shared::files'.
my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
$target .= "/";
$target =~ s/\/\//\//g;
$target .= "/";
$target =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { target => $target }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => {
file => $full_path,
target => $target,
}});
# Wait in case it's still being uploaded.
$anvil->Storage->_wait_if_changing({
file => $full_path,
last_size => $file_size,
});
$anvil->Storage->move_file({
debug => 3,
source_file => $full_path,
@ -975,47 +803,31 @@ sub convert_mimetype
return($say_mimetype);
}
# This looks for any files with the file_type of 'DELETED' and that has an entry in file_locations for this
# machine. Any that are found will have their file deleted and the file_locations entry deleted.
# This looks for any files with the file_type of 'DELETED'. Any that are found will be deleted.
sub check_for_deletes
{
my ($anvil) = @_;
my $query = "
SELECT
a.file_uuid,
b.file_location_uuid,
a.file_directory || '/' || a.file_name AS full_path
FROM
files a,
file_locations b
WHERE
a.file_uuid = b.file_location_file_uuid
AND
a.file_type = 'DELETED'
AND
b.file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
# Get a list of files.
my $query = "SELECT file_uuid, file_directory || '/' || file_name AS full_path FROM files WHERE file_type = 'DELETED';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_uuid = $row->[0];
my $file_location_uuid = $row->[1];
my $full_path = $row->[2];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:file_uuid' => $file_uuid,
's2:file_location_uuid' => $file_location_uuid,
's3:full_path' => $full_path,
my $file_uuid = $row->[0];
my $full_path = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:file_uuid' => $file_uuid,
's2:full_path' => $full_path,
}});
# Get rid of it (if it actually exists) and then remove it from file_locations.
# Get rid of it (if it actually exists).
if (-e $full_path)
{
# Delete it.
@ -1038,10 +850,35 @@ AND
}
}
# Delete the entry from file_locations, if needed.
$query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
# If we're a Striker, check for any file_locations that point to this file and DELETE them.
my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type eq "striker")
{
my $query = "SELECT file_location_uuid FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_location_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
# Delete the entry from file_locations, if needed.
my $query = "DELETE FROM history.file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
$query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
}
}
return(0);
@ -1061,7 +898,7 @@ sub handle_delete
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
$anvil->Job->update_progress({
progress => 0,
progress => 100,
message => "error_0052",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
@ -1072,7 +909,7 @@ sub handle_delete
# We don't do that here...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }});
$anvil->Job->update_progress({
progress => 0,
progress => 100,
message => "error_0053,!!file!".$full_path."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
@ -1126,7 +963,7 @@ AND
# database. This way, if anything goes wrong below, future runs will try (again) to delete the file
# itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt
# to try again.
if (($file_uuid) && ($anvil->data->{switches}{everywhere}))
if ($file_uuid)
{
# Yup.
($file_uuid) = $anvil->Database->insert_or_update_files({
@ -1164,18 +1001,13 @@ AND
}
}
# Now, if I have a file_uuid, delete it from file_locations if it exists.
# Now, if I have a file_uuid, delete it from file_locations if it exists. It's ok if peers in an
# Anvil! haven't deleted yet, as they'll see the file marked as 'DELETED' and purge their local
# copies.
if ($file_uuid)
{
# Delete the entry from file_locations, if needed.
my $query = "
DELETE FROM
file_locations
WHERE
file_location_file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
my $query = "DELETE FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}

Write Preview
Loading…
Cancel
Save