From be84a239244c4bad1989b2b60c798ebe1d5544ca Mon Sep 17 00:00:00 2001 From: Digimer Date: Mon, 8 Aug 2022 21:31:56 -0400 Subject: [PATCH] * There were still references in anvil-manage-files to 'file_locations' -> 'file_location_host_uuid'. Had to rework some logic to get things working. More testing needed, but so far at least the "missing file" function is working again. * Added missing always-available switchs in Get->switches * Create Storage->_wait_if_changing() to check to see if a file's size is changing and, if so, not return until it stops. Signed-off-by: Digimer --- Anvil/Tools/Get.pm | 4 +- Anvil/Tools/Storage.pm | 102 ++++++ man/anvil-manage-files.8 | 38 +++ share/words.xml | 5 +- tools/anvil-manage-files | 664 +++++++++++++++------------------------ 5 files changed, 394 insertions(+), 419 deletions(-) create mode 100644 man/anvil-manage-files.8 diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm index 11c38cc8..6b56ef5d 100644 --- a/Anvil/Tools/Get.pm +++ b/Anvil/Tools/Get.pm @@ -2297,12 +2297,14 @@ sub switches next if $set_switch eq "?"; next if $set_switch eq "h"; next if $set_switch eq "help"; + next if $set_switch eq "log-secure"; + next if $set_switch eq "log-db-transactions"; next if $set_switch eq "raw"; + next if $set_switch eq "resync-db"; next if $set_switch eq "v"; next if $set_switch eq "vv"; next if $set_switch eq "vvv"; next if $set_switch eq "vvvv"; - next if $set_switch eq "log-secure"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set_switch => $set_switch }}); my $found = 0; diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 0626d229..654a2e33 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -46,6 +46,7 @@ my $THIS_FILE = "Storage.pm"; # update_file # write_file # _create_rsync_wrapper +# _wait_if_changing =pod @@ -5151,6 +5152,7 @@ fi"; # Private functions # ############################################################################################################# + =head2 This does the actual work of creating the C<< expect >> wrapper script and returns the path to that wrapper for C<< rsync >> calls. @@ -5227,4 +5229,104 @@ expect eof return($wrapper_script); } + +=head3 _wait_if_changing + +This takes a full path to a file, and watches it for at specified number of seconds to see if the size is changing. If it is, this method waits until the file size stops changing. + +Parameters; + +=head3 file (required) + +This is the full path to the file. If the file is not found, C<< !!error!! >> is returned. + +=head3 delay (optional, default '2') + +This is how long to wait before checking to see if the file has changed. + +=head3 last_size (optional) + +If this is set, it's the first size we compare against. If not passed, the size will be checked. + +=cut +sub _wait_if_changing +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->_create_rsync_wrapper()" }}); + + # Check my parameters. + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $delay = defined $parameter->{delay} ? $parameter->{delay} : ""; + my $last_size = defined $parameter->{last_size} ? $parameter->{last_size} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file => $file, + delay => $delay, + last_size => $last_size, + }}); + + if (not $delay) + { + $delay = 2; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }}); + } + elsif (($delay =~ /\D/) or ($delay == 0)) + { + $delay = 2; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }}); + } + + if (not -e $file) + { + return("!!error!!"); + } + + if (not $last_size) + { + $last_size = (stat($file))[7]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")", + }}); + } + + my $waiting = 1; + while ($waiting) + { + sleep $delay; + my $new_size = (stat($file))[7]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + file => $file, + last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")", + }}); + if ($new_size == $last_size) + { + # Size seems stable + $waiting = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + } + else + { + # Might still be updating, wait. + my $difference = $new_size - $last_size; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0724", variables => { + file => $file, + old_size_bytes => $anvil->Convert->add_commas({number => $last_size}), + old_size_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}), + new_size_bytes => $anvil->Convert->add_commas({number => $new_size}), + new_size_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $new_size}), + difference_bytes => $anvil->Convert->add_commas({number => $difference}), + difference_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $difference}), + }}); + + $last_size = $new_size; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { last_size => $last_size }}); + } + } + + return(0); +} + 1; diff --git a/man/anvil-manage-files.8 b/man/anvil-manage-files.8 new file mode 100644 index 00000000..141a5c03 --- /dev/null +++ b/man/anvil-manage-files.8 @@ -0,0 +1,38 @@ +.\" Manpage for the Anvil! server removal tool +.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions. +.TH anvil-manage-files "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform" +.SH NAME +anvil-manage-files \- This program manages the files sync'ed across machines in the Anvil! cluster +.SH SYNOPSIS +.B anvil-manage-files +\fI\, \/\fR[\fI\,options\/\fR] +.SH DESCRIPTION +This handles synchronizing files, typically ISO used to build new servers and scripts used to manage them, across Anvil! nodes. +.TP +.SH OPTIONS +.TP +\-?, \-h, \fB\-\-help\fR +Show this man page. +.TP +\fB\-\-log-secure\fR +When logging, record sensitive data, like passwords. +.TP +\-v, \-vv, \-vvv +Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data. +.SS "Commands:" +.TP +\fB\-\-delete\fR +This will delete the \fB\-\-file\fR from the entire Anvil! cluster. +.TP +This action is permanent! +.TP +\fB\-\-job-uuid\fR +The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically. +.IP +.SH AUTHOR +Written by Madison Kelly, Alteeve staff and the Anvil! project contributors. +.SH "REPORTING BUGS" +Report bugs to users@clusterlabs.org + + +", "download", "everywhere", "file", "is-script", "job-uuid", "rename", "to \ No newline at end of file diff --git a/share/words.xml b/share/words.xml index 26d75a35..f71947ee 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1664,8 +1664,8 @@ The fingerprint of: [#!variable!machine!#] has changed! Updating it's entry in k The md5sum of file: [#!variable!file!#] failed to match. Discarding the downloaded file. Failed to download: [#!variable!file!#] from: [#!variable!host_name!# (#!variable!ip!#). Will look on other hosts (if any left). The file: [#!variable!file!#] on: [#!variable!host_name!# (#!variable!ip!#]) doesn't match the file we're looking for. -- Wanted; md5sum: [#!variable!file_md5sum!#], size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)] -- Found; md5sum: [#!variable!remote_md5sum!#], size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)] +- Wanted; size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)] +- Found; size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)] We will keep looking. Already searched: [#!variable!host_name!# using another IP address, skipping this IP: [#!variable!ip!#]. Done. @@ -2151,6 +2151,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is: The server: [#!variable!server!#] is ready to boot. The server: [#!variable!server!#] was found to be running already, but it wasn't marked as booted. Marking it as if it just booted to handle any dependent servers. The server: [#!variable!server!#] is configured to stay off, ignoring it. + The file: [#!variable!file!#] needs to be added to the database, but since the last scan it's size grew from: [#!variable!old_size_bytes!# (#!variables!old_size_hr!#)] to: [#!variable!new_size_bytes!# (#!variables!new_size_hr!#)]. A difference of: [#!variable!difference_bytes!# (#!variables!difference_hr!#)]. It might still be being uploaded, so we'll keep checking periodocally until the size stops changing. The host name: [#!variable!target!#] does not resolve to an IP address. diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index 380fa4e5..9197cbed 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -13,10 +13,9 @@ # - If not found anywhere, remove it from 'file_locations' and send an alert. # - If found, check the size. If it differs, recalculate the md5sum. # - 3. If called with '--rename --file --to ', rename the file and update 'files'. -# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If -# also used with '--everywhere', then all copies on all systems we know about will be deleted. This is -# done by registering a job against all known hosts. As such, if this is called and the target file -# doesn't exist, it just clears the job and then exits. +# - 4. If called with '--delete', remove from 'file_locations' and all copies on all systems. This is done by +# registering a job against all known hosts. As such, if this is called and the target file doesn't exist, +# it just clears the job and then exits. # - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit. # # Exit codes; @@ -32,7 +31,7 @@ # - # # NOTE: -# - +# - remove unsyncs, add syncs. # use strict; @@ -52,25 +51,11 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) my $anvil = Anvil::Tools->new(); -$anvil->data->{switches}{'delete'} = ""; -$anvil->data->{switches}{download} = ""; -$anvil->data->{switches}{everywhere} = ""; -$anvil->data->{switches}{file} = ""; -$anvil->data->{switches}{'is-script'} = ""; -$anvil->data->{switches}{'job-uuid'} = ""; -$anvil->data->{switches}{'rename'} = ""; -$anvil->data->{switches}{to} = ""; -$anvil->Get->switches; -$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "switches::delete" => $anvil->data->{switches}{'delete'}, - "switches::download" => $anvil->data->{switches}{download}, - "switches::everywhere" => $anvil->data->{switches}{everywhere}, - "switches::file" => $anvil->data->{switches}{file}, - "switches::is-script" => $anvil->data->{switches}{'is-script'}, - "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'}, - "switches::rename" => $anvil->data->{switches}{'rename'}, - "switches::to" => $anvil->data->{switches}{to}, -}}); +$anvil->Log->level({set => 2}); +$anvil->Log->secure({set => 1}); + +$anvil->Get->switches({list => ["delete", "download", "file", "is-script", "job-uuid", "rename", "remove", "add", "to"], man => $THIS_FILE}); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); # Connect or die $anvil->Database->connect; @@ -171,59 +156,59 @@ sub find_missing_files # What am I? This will impact how missing files are found. my $query = " SELECT - a.file_uuid, - a.file_directory, - a.file_name, - a.file_size, - a.file_md5sum + file_uuid, + file_directory, + file_name, + file_size, + file_md5sum FROM - files a, - hosts b, - file_locations c + files WHERE - b.host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." -AND - a.file_uuid = c.file_location_file_uuid -AND - b.host_uuid = c.file_location_host_uuid + file_type != 'DELETED' ORDER BY - a.file_directory ASC, - a.file_name ASC, - b.host_name ASC + file_directory ASC, + file_name ASC ;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results, count => $count, }}); foreach my $row (@{$results}) { - my $file_uuid = $row->[0]; - my $file_directory = $row->[1]; - my $file_name = $row->[2]; - my $file_size = $row->[3]; - my $file_md5sum = $row->[4]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + my $file_uuid = $row->[0]; + my $file_directory = $row->[1]; + my $file_name = $row->[2]; + my $file_size = $row->[3]; + my $file_md5sum = $row->[4]; + my $full_path = $file_directory."/".$file_name; + $full_path =~ s/\/\//\//g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:file_uuid' => $file_uuid, 's2:file_directory' => $file_directory, 's3:file_name' => $file_name, + 's4:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", + 's5:file_md5sum' => $file_md5sum, + 's6:full_path' => $full_path, }}); - my $test_file = $file_directory."/".$file_name; - $test_file =~ s/\/\//\//g; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_file => $test_file }}); - - if (not -e $test_file) + # If we're a striker, we want all files. + my $host_type = $anvil->Get->host_type(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + if ($host_type eq "striker") { - # Missing file! - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $test_file }}); - - # Find what target, if any, we'll the file from. - my ($found) = find_file($anvil, $file_uuid, $test_file, $file_size, $file_md5sum); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); + if (not -e $full_path) + { + # Missing file! + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $full_path }}); + + # Find what target, if any, we'll the file from. + my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); + } } } @@ -232,338 +217,167 @@ ORDER BY return(0); } -### TODO: Add sorting by preferred target # This looks for a file on another system. The exact order of search depends on what kind of machine we are. sub find_file { my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:file_uuid' => $file_uuid, + 's2:full_path' => $full_path, + 's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", + 's4:file_md5sum' => $file_md5sum, + }}); my $found = 0; - # What are my IPs? - my $local_host = $anvil->Get->short_host_name(); - $anvil->Network->get_ips(); - foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}}) + # We want to search Striker's first, DR hosts second and nodes third. + $anvil->Database->get_hosts; + my $host_order = []; + foreach my $type ("striker", "dr", "node") { - next if not $anvil->data->{network}{$local_host}{interface}{$interface}{ip}; - next if not $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask}; - my $ip = $anvil->data->{network}{$local_host}{interface}{$interface}{ip}; - my $subnet_mask = $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask}; - my $network = $anvil->Network->get_network({ip => $ip, subnet_mask => $subnet_mask}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - 's1:interface' => $interface, - 's2:ip' => $ip, - 's3:subnet_mask' => $subnet_mask, - 's4:network' => $network, - }}); - - my $type = "other"; - my $sort = $interface; - if ($interface =~ /^((?:bc|s|if)n)(\d+)/) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type }}); + foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}}) { - $type = $1; - $sort = $2; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - 's1:type' => $type, - 's2:sort' => $sort, + my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name}; + my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:host_name' => $host_name, + 's2:host_type' => $host_type, + 's3:host_uuid' => $host_uuid, }}); + + next if $host_type ne $type; + next if $host_uuid eq $anvil->Get->host_uuid; + + push @{$host_order}, $host_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }}); } - $anvil->data->{local_ip}{by_network}{$network}{type} = $type; - $anvil->data->{local_ip}{by_network}{$network}{'sort'} = $sort; - $anvil->data->{local_ip}{by_network}{$network}{interface} = $interface; - $anvil->data->{local_ip}{by_network}{$network}{ip} = $ip; - $anvil->data->{local_ip}{by_network}{$network}{subnet_mask} = $subnet_mask; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "s1:local_ip::by_network::${network}::type" => $anvil->data->{local_ip}{by_network}{$network}{type}, - "s1:local_ip::by_network::${network}::sort" => $anvil->data->{local_ip}{by_network}{$network}{'sort'}, - "s2:local_ip::by_network::${network}::interface" => $anvil->data->{local_ip}{by_network}{$network}{interface}, - "s3:local_ip::by_network::${network}::ip" => $anvil->data->{local_ip}{by_network}{$network}{ip}, - "s4:local_ip::by_network::${network}::subnet_mask" => $anvil->data->{local_ip}{by_network}{$network}{subnet_mask}, - }}); } - # Create a hash we can sort through that are on the same subnet_mask as us. - my $query = " -SELECT - a.host_uuid, - a.host_name, - a.host_type, - b.file_directory, - b.file_name, - b.file_size, - b.file_md5sum -FROM - hosts a, - files b, - file_locations c -WHERE - a.host_uuid = c.file_location_host_uuid -AND - b.file_uuid = c.file_location_file_uuid -AND - b.file_uuid = ".$anvil->Database->quote($file_uuid)." -AND - a.host_uuid != ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." -ORDER BY - file_mtime DESC; -;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - results => $results, - count => $count, - }}); - foreach my $row (@{$results}) + # Now search. + my $file_found = 0; + foreach my $search_host_uuid (@{$host_order}) { - my $host_uuid = $row->[0]; - my $host_name = $row->[1]; - my $host_type = $row->[2]; - my $file_directory = $row->[3]; - my $file_name = $row->[4]; - my $file_size = $row->[5]; - my $file_md5sum = $row->[6]; - my $test_file = $file_directory."/".$file_name; - $test_file =~ s/\/\//\//g; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - host_uuid => $host_uuid, - host_name => $host_name, - host_type => $host_type, - file_directory => $file_directory, - file_name => $file_name, - file_size => $file_size, - file_md5sum => $file_md5sum, - test_file => $test_file, + last if $file_found; + my $target_host = $anvil->data->{hosts}{host_uuid}{$search_host_uuid}{short_host_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:search_host_uuid' => $search_host_uuid, + 's2:target_host' => $target_host, }}); - # What IP addresses are on this machine? - my $query = " -SELECT - ip_address_address, - ip_address_subnet_mask -FROM - ip_addresses -WHERE - ip_address_note != 'DELETED' -AND - ip_address_host_uuid = ".$anvil->Database->quote($host_uuid)." -;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); - my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); - my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - results => $results, - count => $count, + my $target_ip = $anvil->Network->find_target_ip({host_uuid => $search_host_uuid}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_ip => $target_ip }}); + next if not $target_ip; + + # See if the file is on the target and, if so, if it matches. + ### NOTE: If we want to use md5sum again, use '--with-md5sum' + my $shell_call = $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path.$anvil->Log->switches; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); + + # Test access + my $access = $anvil->Remote->test_access({target => $target_ip}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); + next if not $access; + + my $remote_size = 0; + my ($output, $error, $return_code) = $anvil->Remote->call({ + shell_call => $shell_call, + target => $target_ip, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + error => $error, + output => $output, + return_code => $return_code, }}); - foreach my $row (@{$results}) + foreach my $line (split/\n/, $output) { - my $ip_address_address = $row->[0]; - my $ip_address_subnet_mask = $row->[1]; - my $network = $anvil->Network->get_network({ip => $ip_address_address, subnet_mask => $ip_address_subnet_mask}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - ip_address_address => $ip_address_address, - ip_address_subnet_mask => $ip_address_subnet_mask, - network => $network, - }}); - - # Are we on the same subnet? - if (exists $anvil->data->{local_ip}{by_network}{$network}) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); + if ($line =~ /^size: \[(\d+)\]$/) { - # We're on the same subnet! - my $type = $anvil->data->{local_ip}{by_network}{$network}{type}; - my $sort = $anvil->data->{local_ip}{by_network}{$network}{'sort'}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "s1:type" => $type, - "s2:sort" => $sort, - }}); - - # Record. - $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip} = $ip_address_address; - $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name} = $host_name; - $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid} = $host_uuid; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "s1:peer_ip::${host_type}::${type}::${sort}::name" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name}, - "s2:peer_ip::${host_type}::${type}::${sort}::ip" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip}, - "s3:peer_ip::${host_type}::${type}::${sort}::host_uuid" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid}, + $remote_size = $1; + $found = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + remote_size => $remote_size, + found => $found, }}); } } - } - - ### TODO: Do this according to what we are. - # Sort through what we've found. - my $file_found = 0; - my $searched_hosts = {}; - foreach my $host_type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}}) - { - last if $file_found; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }}); - foreach my $type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}}) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:found' => $found, + 's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")", + 's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", + }}); + next if not $found; + + if ($remote_size eq $file_size) { - last if $file_found; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }}); - foreach my $sort (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}{$type}}) + # Pull it over! + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => { + file => $full_path, + host_name => $target_host, + ip => $target_ip, + }}); + my $failed = $anvil->Storage->rsync({ + debug => 2, + destination => $full_path, + source => "root\@".$target_ip.":".$full_path, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); + + if (-f $full_path) { - last if $file_found; - my $ip = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip}; - my $name = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name}; - my $host_uuid = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid}; - my $remote_user = "admin"; - my $password = ""; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:host_type' => $host_type, - 's2:type' => $type, - 's3:sort' => $sort, - 's4:name' => $name, - 's5:ip' => $ip, - 's6:host_uuid' => $host_uuid, - 's7:remote_user' => $remote_user, - 's8:password' => $password, - }}); - - if ((exists $searched_hosts->{$name}) && ($searched_hosts->{$name})) - { - # Already searched this host, this is just a different IP. Skip it. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0282", variables => { - host_name => $name, - ip => $ip, - }}); - next; - } - $searched_hosts->{$name} = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "searched_hosts->{$name}" => $searched_hosts->{$name} }}); + # Got it! + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }}); - ### NOTE: There's a bug in Net::SSH2 on RHEL8 where passwordless SSH doesn't - ### work. So for now, we'll manually pull in passwords from the - ### anvil.conf using 'hosts::::password' or - ### 'hosts::::password'. This will be removed when the bug - ### is fixed. - if ((exists $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}) && ($anvil->data->{hosts}{$host_uuid}{password}{$remote_user})) - { - $password = $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }}); - } - elsif ((exists $anvil->data->{hosts}{$name}{password}{$remote_user}) && ($anvil->data->{hosts}{$name}{password}{$remote_user})) - { - $password = $anvil->data->{hosts}{$host_uuid}{password}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }}); - } - - ### NOTE: This might take a while for the call to return if we're md5sum'ing - ### a large file like an ISO. - ### TODO: Should we ask for the size, then follow up with the md5sum for - ### files that are over a certain size? Or will this get called rarely - ### enough in practice that it doesn't really matter? - # If the file is found, we'll parse these out. - my $remote_size = 0; - my $remote_md5sum = ""; - my ($output, $error, $return_code) = $anvil->Remote->call({ - shell_call => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum".$anvil->Log->switches, - remote_user => $remote_user, - password => $password, - target => $ip, - }); + # Verify the md5sum. + my $local_md5sum = $anvil->Get->md5sum({file => $full_path}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - error => $error, - output => $output, - return_code => $return_code, + local_md5sum => $local_md5sum, + file_md5sum => $file_md5sum, }}); - foreach my $line (split/\n/, $output) + if ($local_md5sum eq $file_md5sum) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }}); - if ($line =~ /^size: \[(\d+)\]$/) - { - $remote_size = $1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_size => $remote_size }}); - } - if ($line =~ /^md5sum: \[(.*)\]$/) - { - $remote_md5sum = $1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_md5sum => $remote_md5sum }}); - } - } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:remote_size' => $remote_size, - 's2:file_size' => $file_size, - 's3:remote_md5sum' => $remote_md5sum, - 's4:file_md5sum' => $file_md5sum, - }}); - - ### Do I really need to match sizes if the md5sum is the same? - if (($remote_size eq $file_size) && ($remote_md5sum eq $file_md5sum)) - { - # Pull it over! - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => { - file => $full_path, - host_name => $name, - ip => $ip, - }}); - my $failed = $anvil->Storage->rsync({ - debug => 2, - destination => $full_path, - password => $password, - source => $remote_user."\@".$ip.":".$full_path, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); + # Success! + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }}); - if (-f $full_path) - { - # Got it! - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }}); - - # Verify the md5sum. - my $local_md5sum = $anvil->Get->md5sum({file => $full_path}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - local_md5sum => $local_md5sum, - file_md5sum => $file_md5sum, - }}); - if ($local_md5sum eq $file_md5sum) - { - # Success! - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }}); - - $file_found = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }}); - last; - } - else - { - # Failed. :( - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }}); - unlink $full_path; - } - } - else - { - # Failed to rsync. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => { - file => $full_path, - host_name => $name, - ip => $ip, - }}); - } + $file_found = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }}); + last; } else { - # Doesn't match what we're looking for. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => { - file => $full_path, - host_name => $name, - ip => $ip, - remote_md5sum => $remote_md5sum, - file_md5sum => $file_md5sum, - file_size => $file_size, - say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}), - remote_size => $remote_size, - say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}), - }}); + # Failed. :( + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }}); + unlink $full_path; } } + else + { + # Failed to rsync. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => { + file => $full_path, + host_name => $target_host, + ip => $target_ip, + }}); + } + } + else + { + # Doesn't match what we're looking for. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => { + file => $full_path, + host_name => $target_host, + ip => $target_ip, + file_size => $file_size, + say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}), + remote_size => $remote_size, + say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}), + }}); } } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); - return($found); + return($file_found); } @@ -577,6 +391,7 @@ sub check_incoming { delete $anvil->data->{scan}{directories}; } + # Read any files in '/mnt/shared/incoming'. $anvil->Storage->scan_directory({ debug => 3, @@ -600,13 +415,15 @@ sub check_incoming }}); next if $file_type ne "file"; - my $file_name = $anvil->data->{scan}{directories}{$full_path}{name}; - my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory}; - my $file_size = $anvil->data->{scan}{directories}{$full_path}{size}; - my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime}; - my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; - my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0; - my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable); + my $file_name = $anvil->data->{scan}{directories}{$full_path}{name}; + my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory}; + my $file_size = $anvil->data->{scan}{directories}{$full_path}{size}; + my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime}; + my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; + my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0; + my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable); + my $full_path = $file_directory."/".$file_name; + $full_path =~ s/\/\//\//g; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_name => $file_name, file_size => $file_size, @@ -614,6 +431,7 @@ sub check_incoming file_mimetype => $file_mimetype, file_executable => $file_executable, say_mimetype => $say_mimetype, + full_path => $full_path, }}); # Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum. @@ -626,11 +444,22 @@ sub check_incoming }}); # Calculate the md5sum? - my $file_md5sum = $recorded_md5sum; + my $file_md5sum = $recorded_md5sum; if ((not $file_uuid) or ($file_size != $recorded_size)) { - # Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer - # version down from elsewhere. + # It's possible the file is still uploading, so sleep for 2 seconds and see if the + # size is still changing. + my $last_size = $file_size; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")", + }}); + $anvil->Storage->_wait_if_changing({ + file => $full_path, + last_size => $file_size, + }); + + # Yes, caluclate md5sum, but first, do we have a size mismatch? If so, see if we need + # to pull a newer version down from elsewhere. if (($file_uuid) && ($file_mtime <= $recorded_mtime)) { # We've got an older file, we need to update. @@ -675,14 +504,6 @@ sub check_incoming $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); next if not $file_uuid; - # Make sure we know about this file on this system - my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({ - debug => 3, - file_location_file_uuid => $file_uuid, - file_location_host_uuid => $anvil->data->{sys}{host_uuid}, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_locatiom_uuid => $file_locatiom_uuid }}); - # Are we in the incoming directory? If so, move the file. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { full_path => $full_path, @@ -694,14 +515,21 @@ sub check_incoming # 'path::directories::shared::definitions', otherwise we'll move it to # 'path::directories::shared::files'. my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files}; - $target .= "/"; - $target =~ s/\/\//\//g; + $target .= "/"; + $target =~ s/\/\//\//g; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { target => $target }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => { file => $full_path, target => $target, }}); + + # Wait in case it's still being uploaded. + $anvil->Storage->_wait_if_changing({ + file => $full_path, + last_size => $file_size, + }); + $anvil->Storage->move_file({ debug => 3, source_file => $full_path, @@ -975,47 +803,31 @@ sub convert_mimetype return($say_mimetype); } -# This looks for any files with the file_type of 'DELETED' and that has an entry in file_locations for this -# machine. Any that are found will have their file deleted and the file_locations entry deleted. +# This looks for any files with the file_type of 'DELETED'. Any that are found will be deleted. sub check_for_deletes { my ($anvil) = @_; - my $query = " -SELECT - a.file_uuid, - b.file_location_uuid, - a.file_directory || '/' || a.file_name AS full_path -FROM - files a, - file_locations b -WHERE - a.file_uuid = b.file_location_file_uuid -AND - a.file_type = 'DELETED' -AND - b.file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." -;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); + # Get a list of files. + my $query = "SELECT file_uuid, file_directory || '/' || file_name AS full_path FROM files WHERE file_type = 'DELETED';"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results, count => $count, }}); foreach my $row (@{$results}) { - my $file_uuid = $row->[0]; - my $file_location_uuid = $row->[1]; - my $full_path = $row->[2]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - 's1:file_uuid' => $file_uuid, - 's2:file_location_uuid' => $file_location_uuid, - 's3:full_path' => $full_path, + my $file_uuid = $row->[0]; + my $full_path = $row->[1]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:file_uuid' => $file_uuid, + 's2:full_path' => $full_path, }}); - # Get rid of it (if it actually exists) and then remove it from file_locations. + # Get rid of it (if it actually exists). if (-e $full_path) { # Delete it. @@ -1038,10 +850,35 @@ AND } } - # Delete the entry from file_locations, if needed. - $query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + # If we're a Striker, check for any file_locations that point to this file and DELETE them. + my $host_type = $anvil->Get->host_type(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }}); + if ($host_type eq "striker") + { + my $query = "SELECT file_location_uuid FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $file_location_uuid = $row->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); + + # Delete the entry from file_locations, if needed. + my $query = "DELETE FROM history.file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + + $query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); + } + } } return(0); @@ -1061,7 +898,7 @@ sub handle_delete # Um... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"}); $anvil->Job->update_progress({ - progress => 0, + progress => 100, message => "error_0052", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); @@ -1072,7 +909,7 @@ sub handle_delete # We don't do that here... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }}); $anvil->Job->update_progress({ - progress => 0, + progress => 100, message => "error_0053,!!file!".$full_path."!!", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); @@ -1126,7 +963,7 @@ AND # database. This way, if anything goes wrong below, future runs will try (again) to delete the file # itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt # to try again. - if (($file_uuid) && ($anvil->data->{switches}{everywhere})) + if ($file_uuid) { # Yup. ($file_uuid) = $anvil->Database->insert_or_update_files({ @@ -1164,18 +1001,13 @@ AND } } - # Now, if I have a file_uuid, delete it from file_locations if it exists. + # Now, if I have a file_uuid, delete it from file_locations if it exists. It's ok if peers in an + # Anvil! haven't deleted yet, as they'll see the file marked as 'DELETED' and purge their local + # copies. if ($file_uuid) { # Delete the entry from file_locations, if needed. - my $query = " -DELETE FROM - file_locations -WHERE - file_location_file_uuid = ".$anvil->Database->quote($file_uuid)." -AND - file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." -;"; + my $query = "DELETE FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); }