From cbb441759ef7c553c0d0e9edb8a964e0a7491caf Mon Sep 17 00:00:00 2001 From: Digimer Date: Mon, 15 Aug 2022 23:27:40 -0400 Subject: [PATCH] * Fixed a couple bugs in anvil-manage-files where a file moved from incoming to files or definitions wasn't having the directory updated properly in the database. Also made an explicit check when looking for missing files to check to see if the file exists in another managed directory and, if so and if a striker, update the DB. Signed-off-by: Digimer --- share/words.xml | 1 + tools/anvil-manage-files | 83 +++++++++++++++++++++++++++--------- tools/anvil-provision-server | 2 +- tools/anvil-sync-shared | 72 ------------------------------- 4 files changed, 66 insertions(+), 92 deletions(-) diff --git a/share/words.xml b/share/words.xml index aa3f4939..f9731b44 100644 --- a/share/words.xml +++ b/share/words.xml @@ -2152,6 +2152,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is: The server: [#!variable!server!#] was found to be running already, but it wasn't marked as booted. Marking it as if it just booted to handle any dependent servers. The server: [#!variable!server!#] is configured to stay off, ignoring it. The file: [#!variable!file!#] needs to be added to the database, but since the last scan it's size grew from: [#!variable!old_size_bytes!# (#!variables!old_size_hr!#)] to: [#!variable!new_size_bytes!# (#!variables!new_size_hr!#)]. A difference of: [#!variable!difference_bytes!# (#!variables!difference_hr!#)]. It might still be being uploaded, so we'll keep checking periodocally until the size stops changing. + Found the missing file: [#!variable!file!#] in the directory: [#!variable!directory!#]. Updating the database now. The host name: [#!variable!target!#] does not resolve to an IP address. diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index 083153c8..d07cb7be 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -299,7 +299,40 @@ sub find_file 's4:file_md5sum' => $file_md5sum, }}); - my $found = 0; + # Before we do anything, see if the file is shown as being in a directory other than 'files' in the + # database, but actually in files on disk. If so, update the database. + my $host_type = $anvil->Get->host_type(); + my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory}; + my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name}; + my $files_full_path = $anvil->data->{path}{directories}{shared}{files}."/".$file_name; + $files_full_path =~ s/\/\//\//g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:file_directory' => $file_directory, + 's2:file_name' => $file_name, + 's3:files_full_path' => $files_full_path, + }}); + + if (($full_path ne $files_full_path) && (-e $files_full_path) && ($host_type eq "striker")) + { + ### TODO: Add checks to see if anything other than the directory needs updating. + # It's in 'files' now, update the database. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0725", variables => { + file => $full_path, + directory => $anvil->data->{path}{directories}{shared}{files}, + }}); + $anvil->Database->insert_or_update_files({ + debug => 2, + file_uuid => $file_uuid, + file_name => $anvil->data->{files}{file_uuid}{$file_uuid}{file_name}, + file_directory => $anvil->data->{path}{directories}{shared}{files}, + file_size => $anvil->data->{files}{file_uuid}{$file_uuid}{file_size}, + file_md5sum => $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum}, + file_mtime => $anvil->data->{files}{file_uuid}{$file_uuid}{file_mtime}, + file_type => $anvil->data->{files}{file_uuid}{$file_uuid}{file_type}, + }); + + return(1); + } # We want to search Striker's first, DR hosts second and nodes third. $anvil->Database->get_hosts; @@ -326,14 +359,15 @@ sub find_file } # Now search. - my $file_found = 0; + my $found_on_host = 0; # Found the file on a target + my $file_found = 0; # Actually have the file locally. foreach my $search_host_uuid (@{$host_order}) { last if $file_found; my $target_host = $anvil->data->{hosts}{host_uuid}{$search_host_uuid}{short_host_name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:search_host_uuid' => $search_host_uuid, - 's2:target_host' => $target_host, + 's1:target_host' => $target_host, + 's2:search_host_uuid' => $search_host_uuid, }}); my $target_ip = $anvil->Network->find_target_ip({host_uuid => $search_host_uuid}); @@ -365,20 +399,20 @@ sub find_file $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); if ($line =~ /^size: \[(\d+)\]$/) { - $remote_size = $1; - $found = 1; + $remote_size = $1; + $found_on_host = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - remote_size => $remote_size, - found => $found, + remote_size => $remote_size, + found_on_host => $found_on_host, }}); } } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:found' => $found, - 's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")", - 's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", + 's1:found_on_host' => $found_on_host, + 's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")", + 's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", }}); - next if not $found; + next if not $found_on_host; if ($remote_size eq $file_size) { @@ -447,7 +481,7 @@ sub find_file } } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }}); return($file_found); } @@ -470,16 +504,28 @@ sub check_incoming recursive => 1, }); - my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming}; + my $archives_directory = $anvil->data->{path}{directories}{shared}{archives}; + my $definitions_directory = $anvil->data->{path}{directories}{shared}{definitions}; + my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming}; + my $provision_directory = $anvil->data->{path}{directories}{shared}{provision}; + my $temp_directory = $anvil->data->{path}{directories}{shared}{temp}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + archives_directory => $archives_directory, + definitions_directory => $definitions_directory, + incoming_directory => $incoming_directory, + provision_directory => $provision_directory, + temp_directory => $temp_directory, + }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0264"}); foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}}) { - # Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc) - next if $full_path =~ /^\/mnt\/shared\/temp\//; + # Skip this if it's under a directory managed elsewhere, or that we don't care about. + next if $full_path =~ /^$temp_directory/; + next if $full_path =~ /^$provision_directory/; + next if $full_path =~ /^$definitions_directory/; # Skip if this isn't a file. my $file_type = $anvil->data->{scan}{directories}{$full_path}{type}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { full_path => $full_path, file_type => $file_type, @@ -611,12 +657,11 @@ sub check_incoming }); # Update the file_directory. - my $say_directory =~ s/\/$//; ($file_uuid) = $anvil->Database->insert_or_update_files({ debug => 3, file_uuid => $file_uuid, file_name => $file_name, - file_directory => $say_directory, + file_directory => $target, file_size => $file_size, file_md5sum => $file_md5sum, file_mtime => $file_mtime, diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 4584ab42..76229bfc 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -11,7 +11,7 @@ # TODO: Support cloning; Example # - virt-clone --original-xml /mnt/shared/definitions/.xml --name --file --check path_exists=off # - Make VNC default -# - Don't provision until in the cluster! +# use strict; use warnings; diff --git a/tools/anvil-sync-shared b/tools/anvil-sync-shared index a1cb6965..a802d90b 100755 --- a/tools/anvil-sync-shared +++ b/tools/anvil-sync-shared @@ -77,12 +77,6 @@ if ($anvil->data->{switches}{'job-uuid'}) process_file_mode($anvil); } } -else -{ - # This checks to see if there are any files that need to be pulled, renamed, removed, or mode - # updated. - #run_checks($anvil); -} $anvil->nice_exit({exit_code => 0}); @@ -91,72 +85,6 @@ $anvil->nice_exit({exit_code => 0}); # Functions # ############################################################################################################# -### TODO: Left off here. Run periodic checks for files that need to be pulled/removed, mode changed, -### etc. For 3.0, we'll ignore files people add manually though later we'll probably want to -### auto-sync them. -### NOTE: When finding new files, check the size, sleep for 30 seconds, and check again. If a file's -### size changed, skip it, it's likely still being updated. -### NOTE: This isn't finished, and likely won't be for a while. -sub run_checks -{ - my ($anvil) = @_; - - # First, get a list of files we're expected to have. - $anvil->Database->get_files({debug => 3, include_deleted => 1}); - $anvil->Database->get_file_locations({debug => 3}); - - if ($anvil->Get->host_type eq "striker") - { - # Look for files to add - } - else - { - # Are we an Anvil! member? - my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }}); - - if (not $anvil_uuid) - { - # Nothing to do. - $anvil->nice_exit({exit_code => 0}); - } - - # What files should we have? - foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}}) - { - # TODO: Once per day, calculate the md5sums and compare against the DB. - my $file_uuid = $anvil->data->{files}{file_name}{$file_name}{file_uuid}; - my $file_directory = $anvil->data->{files}{file_name}{$file_name}{file_directory}; - my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size}; - my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type}; - my $file_path = $file_directory."/".$file_name; - my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; - my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - file_uuid => $file_uuid, - file_name => $file_name, - file_directory => $file_directory, - file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")", - file_type => $file_type, - file_path => $file_path, - file_location_uuid => $file_location_uuid, - file_location_active => $file_location_active, - }}); - - if (-e $file_path) - { - # File exists, should it? - } - else - { - # File doesn't exist, should it? - } - } - } - - return(0); -} - sub process_file_mode { my ($anvil) = @_;