seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Fixed a couple bugs in anvil-manage-files where a file moved from incoming to files or definitions wasn't having the directory updated properly in the database. Also made an explicit check when looking for missing files to check to see if the file exists in another managed directory and, if so and if a striker, update the DB.

Browse Source 
Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 2 years ago
parent 7b1771e498
commit cbb441759e
4 changed files with 66 additions and 92 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      share/words.xml
  2. 83
      tools/anvil-manage-files
  3. 2
      tools/anvil-provision-server
  4. 72
      tools/anvil-sync-shared

1
share/words.xml
Unescape View File

@ -2152,6 +2152,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
<key name="log_0722">The server: [#!variable!server!#] was found to be running already, but it wasn't marked as booted. Marking it as if it just booted to handle any dependent servers.</key>
<key name="log_0723">The server: [#!variable!server!#] is configured to stay off, ignoring it.</key>
<key name="log_0724">The file: [#!variable!file!#] needs to be added to the database, but since the last scan it's size grew from: [#!variable!old_size_bytes!# (#!variables!old_size_hr!#)] to: [#!variable!new_size_bytes!# (#!variables!new_size_hr!#)]. A difference of: [#!variable!difference_bytes!# (#!variables!difference_hr!#)]. It might still be being uploaded, so we'll keep checking periodocally until the size stops changing.</key>
<key name="log_0725">Found the missing file: [#!variable!file!#] in the directory: [#!variable!directory!#]. Updating the database now.</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

83
tools/anvil-manage-files
Unescape View File

@ -299,7 +299,40 @@ sub find_file
's4:file_md5sum' => $file_md5sum,
}});
my $found = 0;
# Before we do anything, see if the file is shown as being in a directory other than 'files' in the
# database, but actually in files on disk. If so, update the database.
my $host_type = $anvil->Get->host_type();
my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
my $files_full_path = $anvil->data->{path}{directories}{shared}{files}."/".$file_name;
$files_full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:file_directory' => $file_directory,
's2:file_name' => $file_name,
's3:files_full_path' => $files_full_path,
}});
if (($full_path ne $files_full_path) && (-e $files_full_path) && ($host_type eq "striker"))
{
### TODO: Add checks to see if anything other than the directory needs updating.
# It's in 'files' now, update the database.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0725", variables => {
file => $full_path,
directory => $anvil->data->{path}{directories}{shared}{files},
}});
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{files}{file_uuid}{$file_uuid}{file_name},
file_directory => $anvil->data->{path}{directories}{shared}{files},
file_size => $anvil->data->{files}{file_uuid}{$file_uuid}{file_size},
file_md5sum => $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum},
file_mtime => $anvil->data->{files}{file_uuid}{$file_uuid}{file_mtime},
file_type => $anvil->data->{files}{file_uuid}{$file_uuid}{file_type},
});
return(1);
}
# We want to search Striker's first, DR hosts second and nodes third.
$anvil->Database->get_hosts;
@ -326,14 +359,15 @@ sub find_file
}
# Now search.
my $file_found = 0;
my $found_on_host = 0; # Found the file on a target
my $file_found = 0; # Actually have the file locally.
foreach my $search_host_uuid (@{$host_order})
{
last if $file_found;
my $target_host = $anvil->data->{hosts}{host_uuid}{$search_host_uuid}{short_host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:search_host_uuid' => $search_host_uuid,
's2:target_host' => $target_host,
's1:target_host' => $target_host,
's2:search_host_uuid' => $search_host_uuid,
}});
my $target_ip = $anvil->Network->find_target_ip({host_uuid => $search_host_uuid});
@ -365,20 +399,20 @@ sub find_file
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /^size: \[(\d+)\]$/)
{
$remote_size = $1;
$found = 1;
$remote_size = $1;
$found_on_host = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
remote_size => $remote_size,
found => $found,
remote_size => $remote_size,
found_on_host => $found_on_host,
}});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:found' => $found,
's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")",
's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
's1:found_on_host' => $found_on_host,
's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")",
's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
}});
next if not $found;
next if not $found_on_host;
if ($remote_size eq $file_size)
{
@ -447,7 +481,7 @@ sub find_file
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
return($file_found);
}
@ -470,16 +504,28 @@ sub check_incoming
recursive => 1,
});
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
my $archives_directory = $anvil->data->{path}{directories}{shared}{archives};
my $definitions_directory = $anvil->data->{path}{directories}{shared}{definitions};
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
my $provision_directory = $anvil->data->{path}{directories}{shared}{provision};
my $temp_directory = $anvil->data->{path}{directories}{shared}{temp};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
archives_directory => $archives_directory,
definitions_directory => $definitions_directory,
incoming_directory => $incoming_directory,
provision_directory => $provision_directory,
temp_directory => $temp_directory,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0264"});
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
{
# Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc)
next if $full_path =~ /^\/mnt\/shared\/temp\//;
# Skip this if it's under a directory managed elsewhere, or that we don't care about.
next if $full_path =~ /^$temp_directory/;
next if $full_path =~ /^$provision_directory/;
next if $full_path =~ /^$definitions_directory/;
# Skip if this isn't a file.
my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path,
file_type => $file_type,
@ -611,12 +657,11 @@ sub check_incoming
});
# Update the file_directory.
my $say_directory =~ s/\/$//;
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $say_directory,
file_directory => $target,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,

2
tools/anvil-provision-server
Unescape View File

@ -11,7 +11,7 @@
# TODO: Support cloning; Example
# - virt-clone --original-xml /mnt/shared/definitions/<source>.xml --name <new_server> --file <new_server's_drbd_path> --check path_exists=off
# - Make VNC default
# - Don't provision until in the cluster!
#
use strict;
use warnings;

72
tools/anvil-sync-shared
Unescape View File

@ -77,12 +77,6 @@ if ($anvil->data->{switches}{'job-uuid'})
process_file_mode($anvil);
}
}
else
{
# This checks to see if there are any files that need to be pulled, renamed, removed, or mode
# updated.
#run_checks($anvil);
}
$anvil->nice_exit({exit_code => 0});
@ -91,72 +85,6 @@ $anvil->nice_exit({exit_code => 0});
# Functions #
#############################################################################################################
### TODO: Left off here. Run periodic checks for files that need to be pulled/removed, mode changed,
### etc. For 3.0, we'll ignore files people add manually though later we'll probably want to
### auto-sync them.
### NOTE: When finding new files, check the size, sleep for 30 seconds, and check again. If a file's
### size changed, skip it, it's likely still being updated.
### NOTE: This isn't finished, and likely won't be for a while.
sub run_checks
{
my ($anvil) = @_;
# First, get a list of files we're expected to have.
$anvil->Database->get_files({debug => 3, include_deleted => 1});
$anvil->Database->get_file_locations({debug => 3});
if ($anvil->Get->host_type eq "striker")
{
# Look for files to add
}
else
{
# Are we an Anvil! member?
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
if (not $anvil_uuid)
{
# Nothing to do.
$anvil->nice_exit({exit_code => 0});
}
# What files should we have?
foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
{
# TODO: Once per day, calculate the md5sums and compare against the DB.
my $file_uuid = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
my $file_directory = $anvil->data->{files}{file_name}{$file_name}{file_directory};
my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size};
my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type};
my $file_path = $file_directory."/".$file_name;
my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
file_type => $file_type,
file_path => $file_path,
file_location_uuid => $file_location_uuid,
file_location_active => $file_location_active,
}});
if (-e $file_path)
{
# File exists, should it?
}
else
{
# File doesn't exist, should it?
}
}
}
return(0);
}
sub process_file_mode
{
my ($anvil) = @_;

Write Preview
Loading…
Cancel
Save