seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

This should fix the tracking of what files belong where, using the new DR links system. It also should finish (though testing is still needed) the serial rsync issue.

Browse Source 
* Created Database->track_files() as a dedicated method as trying to verify the existence of file_locations during Database->load_anvils() was fragile and prone to recursive loops.
* Updated Database->insert_or_update_file_locations() to take an anvil_uuid and recursively call for each host, to maintain compatibility with the old ways, and make it simpler to add an entry for both sub-nodes in an Anvil!.
* Created Storage->push_file() that takes a file and rsync's it to all other machines, or creates a job for the file to be pulled if the target can't be accessed.
* Updated anvil-manage-files and anvil-sync-shared to use the new Storage->push_files and Database->track_files methods.

Signed-off-by: digimer <mkelly@alteeve.ca>
main
digimer 2 years ago
parent 645f54ab89
commit 254f7ef4e2
5 changed files with 645 additions and 326 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 468
      Anvil/Tools/Database.pm
  2. 266
      Anvil/Tools/Storage.pm
  3. 5
      notes
  4. 61
      tools/anvil-manage-files
  5. 171
      tools/anvil-sync-shared

468
Anvil/Tools/Database.pm
Unescape View File

@ -97,6 +97,7 @@ my $THIS_FILE = "Database.pm";
# refresh_timestamp
# resync_databases
# shutdown
# track_files
# update_host_status
# write
# _add_to_local_config
@ -401,8 +402,8 @@ sub check_file_locations
# Get all the Anvil! systems we know of.
$anvil->Database->get_hosts({debug => $debug});
$anvil->Database->get_files({debug => $debug});
$anvil->Database->get_file_locations({debug => $debug});
#$anvil->Database->get_files({debug => $debug});
#$anvil->Database->get_file_locations({debug => $debug});
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{hosts}{host_name}})
{
@ -2830,135 +2831,6 @@ WHERE
}});
}
}
# Track the files on this Anvil!
foreach my $file_location_uuid (keys %{$anvil->data->{file_locations}{file_location_uuid}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
# If either node is set have this file, make sure both are.
my $file_uuid = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid};
my $anvil_needs_file = 0;
$anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid} = 0;
$anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid} = 0;
if ((exists $anvil->data->{file_locations}{host_uuid}{$anvil_node1_host_uuid}{file_uuid}{$file_uuid}) &&
($anvil->data->{file_locations}{host_uuid}{$anvil_node1_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}))
{
my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$anvil_node1_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid,
file_location_active => $file_location_active,
}});
if (not $file_location_active)
{
$anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid} = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"needs_file::${file_uuid}::${anvil_node1_host_uuid}" => $anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid},
}});
}
}
if ((exists $anvil->data->{file_locations}{host_uuid}{$anvil_node2_host_uuid}{file_uuid}{$file_uuid}) &&
($anvil->data->{file_locations}{host_uuid}{$anvil_node2_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}))
{
my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$anvil_node2_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid,
file_location_active => $file_location_active,
}});
if (not $file_location_active)
{
$anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid} = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"needs_file::${file_uuid}::${anvil_node2_host_uuid}" => $anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid},
}});
}
}
# If either node has the file, both nodes (and all DRs) need the file.
if (($anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid}) or ($anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid}))
{
$anvil_needs_file = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_needs_file => $anvil_needs_file }});
}
foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid)
{
# If the Anvil! needs the file, make sure that both nodes and any connected DRs have it as well.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
host_uuid => $host_uuid,
}});
if ($anvil_needs_file)
{
if (not $anvil->{needs_file}{$file_uuid}{$host_uuid})
{
$anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_file_uuid => $file_location_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
}
}
}
# If the Anvil! node needs the file, record it as being on this Anvil! node.
if ($anvil_needs_file)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"file_locations::file_location_uuid::${file_location_uuid}::file_location_host_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid},
"file_locations::file_location_uuid::${file_location_uuid}::file_location_active" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active},
}});
next if not exists $anvil->data->{files}{file_uuid}{$file_uuid};
my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_uuid => $file_uuid,
file_name => $file_name,
}});
# If the file was deleted, this won't exist
next if not exists $anvil->data->{files}{file_uuid}{$file_uuid};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name} = $file_name;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_size};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_size" => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}}).")",
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type},
}});
# Make it so that we can list the files by name.
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid},
}});
# Make sure linked DR hosts have this file, also.
foreach my $host_uuid (keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}})
{
my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
if (not $file_location_active)
{
$anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_file_uuid => $file_location_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
}
}
}
}
}
return(0);
@ -3372,6 +3244,18 @@ FROM
"file_locations::file_location_uuid::${file_location_uuid}::modified_date" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date},
}});
### TODO: Remove this when the WebUI is updated.
# If this host is a node in an Anvil!, set the old 'file_location_anvil_uuid' to maintain
# backwards compatibility.
if ((exists $anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}) &&
($anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}{anvil_uuid}))
{
$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_anvil_uuid} = $anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}{anvil_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"file_locations::file_location_uuid::${file_location_uuid}::file_location_anvil_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_anvil_uuid},
}});
}
# Make it easy to find files by anvil and file UUID.
$anvil->data->{file_locations}{host_uuid}{$file_location_host_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid} = $file_location_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
@ -8002,6 +7886,12 @@ Parameters;
If not passed, a check will be made to see if an existing entry is found for C<< file_location_file_uuid >>. If found, that entry will be updated. If not found, a new record will be inserted.
=head3 file_location_anvil_uuid (required)
This is the C<< anvils >> -> C<< anvil_uuid >> being referenced. This works by figuring out which hosts are a member of the Anvil! node, and which DR hosts are linked, and makes a recursive call to this method for each of their C<< hosts >> -> C<< host_uuid >>.
B<< Note >>: When this is used, a comma-separated list of C<< host_uuid=file_location_uuid >> is returned.
=head3 file_location_file_uuid (required)
This is the C<< files >> -> C<< file_uuid >> being referenced.
@ -8025,21 +7915,23 @@ sub insert_or_update_file_locations
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }});
my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : "";
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : "";
my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : "";
my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : "";
my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : "";
my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : "";
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : "";
my $file_location_anvil_uuid = defined $parameter->{file_location_anvil_uuid} ? $parameter->{file_location_anvil_uuid} : "";
my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : "";
my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : "";
my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
uuid => $uuid,
file => $file,
line => $line,
file_location_uuid => $file_location_uuid,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $file_location_host_uuid,
file_location_active => $file_location_active,
uuid => $uuid,
file => $file,
line => $line,
file_location_uuid => $file_location_uuid,
file_location_anvil_uuid => $file_location_anvil_uuid,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $file_location_host_uuid,
file_location_active => $file_location_active,
}});
if (not $file_location_file_uuid)
@ -8061,6 +7953,59 @@ sub insert_or_update_file_locations
return("");
}
# If we've got an Anvil! uuid, find out the hosts and DR links connected to the Anvil! are found and
# this method is recursively called for each host.
if ($file_location_anvil_uuid)
{
$anvil->Database->get_anvils({debug => $debug});
if (not exists $anvil->data->{anvils}{anvil_uuid}{$file_location_anvil_uuid})
{
# Bad Anvil! UUID.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0169", variables => { anvil_uuid => $file_location_anvil_uuid }});
return("");
}
my $hosts = ();
push @{$hosts}, $anvil->data->{anvils}{anvil_uuid}{$file_location_anvil_uuid}{anvil_node1_host_uuid};
push @{$hosts}, $anvil->data->{anvils}{anvil_uuid}{$file_location_anvil_uuid}{anvil_node2_host_uuid};
if (exists $anvil->data->{dr_links}{by_anvil_uuid}{$file_location_anvil_uuid})
{
foreach my $dr_link_host_uuid (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$file_location_anvil_uuid}{dr_link_host_uuid}})
{
my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$file_location_anvil_uuid}{dr_link_host_uuid}{$dr_link_host_uuid}{dr_link_uuid};
my $dr_link_note = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note};
my $dr_link_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}{short_host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:dr_link_host_uuid" => $dr_link_host_uuid,
"s2:dr_link_uuid" => $dr_link_uuid,
"s3:dr_link_note" => $dr_link_note,
"s4:dr_link_short_host_name" => $dr_link_short_host_name,
}});
next if $dr_link_note eq "DELETED";
push @{$hosts}, $dr_link_host_uuid;
}
}
my $file_location_uuids = "";
foreach my $host_uuid (@{$hosts})
{
my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => $file_location_active,
});
$file_location_uuids .= $host_uuid."=".$file_location_uuid.",";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid,
file_location_uuids => $file_location_uuids,
}});
}
$file_location_uuids =~ s/,$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuids => $file_location_uuids }});
return($file_location_uuids);
}
# If we don't have a UUID, see if we can find one for the given md5sum.
if (not $file_location_uuid)
{
@ -8124,7 +8069,7 @@ INSERT INTO
SELECT
file_location_file_uuid,
file_location_host_uuid,
file_location_active,
file_location_active
FROM
file_locations
WHERE
@ -17575,6 +17520,237 @@ AND
}
=head2 track_file
This looks at all files in the database, and then for all Anvil! systems and linked DR hosts, ensures that there's a corresponding C<< file_locations >> entry.
This method takes no parameters.
=cut
sub track_files
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->track_files()" }});
my $anvils = keys %{$anvil->data->{anvils}{anvil_name}};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvils => $anvils }});
if (not $anvils)
{
$anvil->Database->get_anvils({debug => $debug});
}
my $files = keys %{$anvil->data->{files}{file_uuid}};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { files => $files }});
if (not $files)
{
$anvil->Database->get_files({debug => $debug});
}
my $file_locations = keys %{$anvil->data->{file_locations}{file_location_uuid}};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_locations => $file_locations }});
if (not $file_locations)
{
$anvil->Database->get_file_locations({debug => $debug});
}
my $dr_link_uuid = keys %{$anvil->data->{dr_links}{dr_link_uuid}};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { dr_link_uuid => $dr_link_uuid }});
if (not $dr_link_uuid)
{
$anvil->Database->get_dr_links({debug => $debug});
}
foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
{
my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
my $anvil_description = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_description};
my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:anvil_name' => $anvil_name,
's2:anvil_uuid' => $anvil_uuid,
's3:anvil_description' => $anvil_description,
's4:anvil_node1_host_uuid' => $anvil_node1_host_uuid,
's5:anvil_node2_host_uuid' => $anvil_node2_host_uuid,
}});
# Loop through all files and see if there's a corresponding file_location for each sub-node.
my $reload = 0;
foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
{
my $file_uuid = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:file_name' => $file_name,
's2:file_uuid' => $file_uuid,
}});
foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
if ((not exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}) or
($anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}))
{
# Add it
$reload = 1;
my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
reload => $reload,
file_location_uuid => $file_location_uuid,
}});
}
}
}
if ($reload)
{
$anvil->Database->get_file_locations({debug => $debug});
}
# Track the files on this Anvil!
foreach my $file_location_uuid (keys %{$anvil->data->{file_locations}{file_location_uuid}})
{
my $file_uuid = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid};
my $file_type = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type};
my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid,
file_uuid => $file_uuid,
file_type => $file_type,
file_name => $file_name,
}});
next if $file_type eq "DELETED";
### TODO - Left off here, not adding DR links.
my $anvil_needs_file = 0;
foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
if ((exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}) &&
($anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}))
{
my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid,
file_location_active => $file_location_active,
}});
if ($file_location_active)
{
$anvil_needs_file = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_needs_file => $anvil_needs_file }});
}
}
}
# If either node wanted the file, both nodes and all linked DRs need it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_needs_file => $anvil_needs_file }});
if ($anvil_needs_file)
{
# Update the hosts
foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
}
# Make sure linked DR hosts have this file, also.
foreach my $host_uuid (keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}})
{
my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
if (not $file_location_active)
{
my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
}
}
# If the file was deleted, this won't exist
next if not exists $anvil->data->{files}{file_uuid}{$file_uuid};
# Record that this Anvil! node has this file.
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name} = $file_name;
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_size};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_size" => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}}).")",
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type},
}});
# Make it so that we can list the files by name.
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid},
}});
# Make sure linked DR hosts have this file, also.
foreach my $host_uuid (keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}})
{
my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
host_name => $host_name,
host_uuid => $host_uuid,
}});
my $file_location_uuid = "";
my $file_location_active = 0;
if (exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid})
{
$file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
$file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid,
file_location_active => $file_location_active,
}});
if (not $file_location_active)
{
my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
}
}
}
}
}
return(0);
}
=head2 update_host_status
This is a variant on C<< insert_or_update_hosts >> designed only to update the power status of a host.

266
Anvil/Tools/Storage.pm
Unescape View File

@ -35,6 +35,7 @@ my $THIS_FILE = "Storage.pm";
# move_file
# parse_df
# parse_lsblk
# push_file
# read_config
# read_file
# read_mode
@ -3451,6 +3452,271 @@ sub parse_lsblk
}
=head2 push_file
This takes a file and pushes it to all other machines in the cluster, serially. For machines that can't be accessed, a job is registered to pull the file.
If C<< switches::job-uuid >> is set, the corresponding job will be updated. The progress assumes that C<< sys::progress >> is set.
Parameters;
=head3 file (required)
This is the source file to copy from locally and push it to all peers' C<< /mnt/shared/files/ >> directory.
=cut
sub push_file
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->push_file()" }});
# Setup default values
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file => $file,
file_uuid => $file_uuid,
}});
if (not $file)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->push_file()", parameter => "file" }});
return("!!error!!");
}
if (not -f $file)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0105", variables => { file => $file }});
return("!!error!!");
}
$anvil->Database->get_files({debug => $debug});
my $file_size = 0;
my ($file_directory, $file_name) = ($file =~ /^(.*)\/(.*?)$/);
$file_directory =~ s/\/$//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:file_directory' => $file_directory,
's2:file_name' => $file_name,
}});
if (not $file_uuid)
{
# Can we find the file?
foreach my $this_file_uuid (keys %{$anvil->data->{files}{file_uuid}})
{
my $this_file_name = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_name};
my $this_file_directory = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_directory};
$this_file_directory =~ s/\/$//g;
my $this_file_size = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_size};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
this_file_uuid => $this_file_uuid,
this_file_directory => $this_file_directory,
this_file_name => $this_file_name,
this_file_size => $this_file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_file_size}).")",
}});
if (($file_name eq $this_file_name) &&
($file_directory eq $this_file_directory))
{
# Found it.
$file_uuid = $this_file_uuid;
$file_size = $this_file_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_uuid => $file_uuid,
file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
}});
last;
}
}
}
if (not $file_uuid)
{
$file_size = (stat($file))[7];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
}});
}
# Now copy this to our peers. We're going to do this serially so that we don't overwhelm the system,
# Any hosts not currently online will have a job registered.
$anvil->Database->get_hosts;
my $host_uuid = $anvil->Get->host_uuid();
my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name};
my $target_directory = $anvil->data->{path}{directories}{shared}{files}."/";
foreach my $do_host_type ("striker", "node", "dr")
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { do_host_type => $do_host_type }});
foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
{
my $target_host_uuid = $anvil->data->{sys}{hosts}{by_name}{$target_host_name};
my $target_host_type = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{host_type};
my $target_short_host_name = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{short_host_name};
next if $target_host_uuid eq $host_uuid;
next if $target_host_type ne $do_host_type;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:target_host_name' => $target_host_name,
's2:target_host_uuid' => $target_host_uuid,
's3:target_host_type' => $target_host_type,
's4:target_short_host_name' => $target_short_host_name,
}});
my $matches = $anvil->Network->find_access({
debug => 2,
target => $target_short_host_name,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }});
next if not $matches;
next if $matches =~ /\D/;
# Find a matching IP.
# We prefer to use least to most used networks, with the IFN being the last choice.
my $copied = 0;
foreach my $network ("mn", "bcn", "sn", "ifn")
{
next if $copied;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network => $network }});
foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}})
{
next if $copied;
next if $network_name !~ /^$network/i;
my $local_interface = $anvil->data->{network_access}{$network_name}{local_interface};
my $local_speed = $anvil->data->{network_access}{$network_name}{local_speed};
my $local_ip_address = $anvil->data->{network_access}{$network_name}{local_ip_address};
my $local_subnet_mask = $anvil->data->{network_access}{$network_name}{local_subnet_mask};
my $target_interface = $anvil->data->{network_access}{$network_name}{target_interface};
my $target_speed = $anvil->data->{network_access}{$network_name}{target_speed};
my $target_ip_address = $anvil->data->{network_access}{$network_name}{target_ip_address};
my $target_subnet_mask = $anvil->data->{network_access}{$network_name}{target_subnet_mask};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:local_interface' => $local_interface,
's2:local_speed' => $local_speed,
's3:local_ip_address' => $local_ip_address,
's4:local_subnet_mask' => $local_subnet_mask,
's5:target_interface' => $target_interface,
's6:target_speed' => $target_speed,
's7:target_ip_address' => $target_ip_address,
's8:target_subnet_mask' => $target_subnet_mask,
}});
my $access = $anvil->Remote->test_access({target => $target_ip_address});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
if ($access)
{
### Rsync!
# Estimate how long this will take. First, get the speed in
# Mbps and turn it into bytes per second, then into bytes per
# second. We'll take 10% off, then calculate how many seconds
# the copy will take.
my $link_mbps = $target_speed > $local_speed ? $target_speed : $local_speed;
my $link_bps = $link_mbps * 1000000;
my $link_bytes_sec = int($link_bps / 8);
my $adjusted_byptes_sec = int($link_bytes_sec * 0.9);
my $copy_seconds = int($file_size / $adjusted_byptes_sec);
my $say_copy_time = $anvil->Convert->time({'time' => $copy_seconds, translate => 1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:link_mbps' => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}),
's2:link_bps' => $anvil->Convert->add_commas({number => $link_bps})." ".$anvil->Words->string({string => "#!string!suffix_0048!#"}),
's3:link_bytes_sec' => $anvil->Convert->add_commas({number => $link_bytes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}),
's4:adjusted_byptes_sec' => $anvil->Convert->add_commas({number => $adjusted_byptes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}),
's5:copy_seconds' => $anvil->Convert->add_commas({number => $copy_seconds})." ".$anvil->Words->string({string => "#!string!suffix_0007!#"}),
's6:say_copy_time' => $say_copy_time,
}});
my $variables = {
host => $target_short_host_name,
network => $network_name,
ip_address => $target_ip_address,
source_file => $file,
target_directory => $target_directory,
size => $anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}),
link_speed => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}),
eta_copy_time => $say_copy_time,
};
$anvil->data->{sys}{progress} += 2;
$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
$anvil->Job->update_progress({
progress => $anvil->data->{sys}{progress},
message => "message_0195",
variables => $variables,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => $variables});
my $problem = $anvil->Storage->rsync({
debug => 2,
source => $file,
destination => "root\@".$target_ip_address.":".$target_directory,
try_again => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if (not $problem)
{
$copied = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }});
$anvil->data->{sys}{progress} += 5;
$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
$anvil->Job->update_progress({
progress => $anvil->data->{sys}{progress},
message => "message_0310",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0310"});
}
}
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
if ($file_uuid)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }});
if (not $copied)
{
# Failed to connect, register a job instead.
my $variables = { host => $target_host_name };
$anvil->data->{sys}{progress} += 5;
$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
$anvil->Job->update_progress({
progress => $anvil->data->{sys}{progress},
message => "message_0196",
variables => $variables,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => $variables});
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE,
line => __LINE__,
job_command => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches,
job_data => "file_uuid=".$file_uuid,
job_name => "storage::pull_file",
job_title => "job_0132",
job_description => "job_0133",
job_progress => 0,
job_host_uuid => $target_host_uuid,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
}
# Mark the file as being on this host
if ($do_host_type ne "striker")
{
my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({
debug => 2,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
}
}
}
}
return(0);
}
=head2 read_config
This method is used to read 'Anvil::Tools' style configuration files. These configuration files are in the format:

5
notes
Unescape View File

@ -1,4 +1,9 @@
Common queries;
* SELECT a.job_uuid, b.host_name, a.job_command, a.job_data, a.job_progress, a.job_status FROM jobs a, hosts b WHERE a.job_host_uuid = b.host_uuid AND a.job_progress != 100;
* SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC;
* SELECT a.dr_link_uuid, b.host_name, c.anvil_name, a.dr_link_note FROM dr_links a, hosts b, anvils c WHERE a.dr_link_host_uuid = b.host_uuid AND a.dr_link_anvil_uuid = c.anvil_uuid ORDER BY c.anvil_name ASC, b.host_name ASC;
When pairing Striker, make sure new config goes to all known nodes!

61
tools/anvil-manage-files
Unescape View File

@ -564,7 +564,8 @@ sub check_incoming
}});
# Calculate the md5sum?
my $file_md5sum = $recorded_md5sum;
my $added_to_db = 0;
my $file_md5sum = $recorded_md5sum;
if ((not $file_uuid) or ($file_size != $recorded_size))
{
# It's possible the file is still uploading, so sleep for a bit and see if the
@ -608,7 +609,7 @@ sub check_incoming
# Insert or update the files entry.
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3,
debug => 2,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
@ -617,15 +618,21 @@ sub check_incoming
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
if ($file_uuid)
{
$added_to_db = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { added_to_db => $added_to_db }});
}
}
# If we still don't have a file UUID for some reason, skip this file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
next if not $file_uuid;
# Are we in the incoming directory? If so, move the file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
full_path => $full_path,
incoming_directory => $incoming_directory,
}});
@ -640,28 +647,34 @@ sub check_incoming
my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files};
$target .= "/";
$target =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { target => $target }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => {
file => $full_path,
target => $target,
}});
# Wait in case it's still being uploaded.
$anvil->Storage->_wait_if_changing({
file => $full_path,
last_size => $file_size,
});
$anvil->Storage->move_file({
debug => 3,
my $problem = $anvil->Storage->move_file({
debug => 2,
source_file => $full_path,
target_file => $target,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if (not $problem)
{
$full_path = $target."/".$file_name;
$file_directory = $target;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_directory => $file_directory,
full_path => $full_path,
}});
}
# Update the file_directory.
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3,
$target =~ s/\/$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }});
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $target,
@ -670,7 +683,20 @@ sub check_incoming
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
}
if ($added_to_db)
{
# Push it to all other hosts now.
$anvil->Storage->push_file({
debug => 2,
file => $full_path,
file_uuid => $file_uuid,
});
# Call track_files, it'll make sure the file_locations are setup.
$anvil->Database->track_files({debug => 2});
}
}
@ -729,6 +755,7 @@ WHERE
{
$query .= " file_name = ".$anvil->Database->quote($file_name)."\n";
}
$query .= ";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};

171
tools/anvil-sync-shared
Unescape View File

@ -574,169 +574,14 @@ sub process_incoming_file
# Now copy this to our peers. We're going to do this serially so that we don't overwhelm the system,
# Any hosts not currently online will have a job registered.
$anvil->Database->get_hosts;
my $host_uuid = $anvil->Get->host_uuid();
my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name};
my $target_directory = $anvil->data->{path}{directories}{shared}{files}."/";
foreach my $do_host_type ("striker", "node", "dr")
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { do_host_type => $do_host_type }});
foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
{
my $target_host_uuid = $anvil->data->{sys}{hosts}{by_name}{$target_host_name};
my $target_host_type = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{host_type};
my $target_short_host_name = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{short_host_name};
next if $target_host_uuid eq $host_uuid;
next if $target_host_type ne $do_host_type;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:target_host_name' => $target_host_name,
's2:target_host_uuid' => $target_host_uuid,
's3:target_host_type' => $target_host_type,
's4:target_short_host_name' => $target_short_host_name,
}});
my $matches = $anvil->Network->find_access({
debug => 2,
target => $target_short_host_name,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }});
next if not $matches;
next if $matches =~ /\D/;
# Find a matching IP.
# We prefer to use least to most used networks, with the IFN being the last choice.
my $copied = 0;
foreach my $network ("mn", "bcn", "sn", "ifn")
{
next if $copied;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network => $network }});
foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}})
{
next if $copied;
next if $network_name !~ /^$network/i;
my $local_interface = $anvil->data->{network_access}{$network_name}{local_interface};
my $local_speed = $anvil->data->{network_access}{$network_name}{local_speed};
my $local_ip_address = $anvil->data->{network_access}{$network_name}{local_ip_address};
my $local_subnet_mask = $anvil->data->{network_access}{$network_name}{local_subnet_mask};
my $target_interface = $anvil->data->{network_access}{$network_name}{target_interface};
my $target_speed = $anvil->data->{network_access}{$network_name}{target_speed};
my $target_ip_address = $anvil->data->{network_access}{$network_name}{target_ip_address};
my $target_subnet_mask = $anvil->data->{network_access}{$network_name}{target_subnet_mask};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:local_interface' => $local_interface,
's2:local_speed' => $local_speed,
's3:local_ip_address' => $local_ip_address,
's4:local_subnet_mask' => $local_subnet_mask,
's5:target_interface' => $target_interface,
's6:target_speed' => $target_speed,
's7:target_ip_address' => $target_ip_address,
's8:target_subnet_mask' => $target_subnet_mask,
}});
my $access = $anvil->Remote->test_access({target => $target_ip_address});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
if ($access)
{
### Rsync!
# Estimate how long this will take. First, get the speed in
# Mbps and turn it into bytes per second, then into bytes per
# second. We'll take 10% off, then calculate how many seconds
# the copy will take.
my $link_mbps = $target_speed > $local_speed ? $target_speed : $local_speed;
my $link_bps = $link_mbps * 1000000;
my $link_bytes_sec = int($link_bps / 8);
my $adjusted_byptes_sec = int($link_bytes_sec * 0.9);
my $copy_seconds = int($file_size / $adjusted_byptes_sec);
my $say_copy_time = $anvil->Convert->time({'time' => $copy_seconds, translate => 1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:link_mbps' => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}),
's2:link_bps' => $anvil->Convert->add_commas({number => $link_bps})." ".$anvil->Words->string({string => "#!string!suffix_0048!#"}),
's3:link_bytes_sec' => $anvil->Convert->add_commas({number => $link_bytes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}),
's4:adjusted_byptes_sec' => $anvil->Convert->add_commas({number => $adjusted_byptes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}),
's5:copy_seconds' => $anvil->Convert->add_commas({number => $copy_seconds})." ".$anvil->Words->string({string => "#!string!suffix_0007!#"}),
's6:say_copy_time' => $say_copy_time,
}});
my $variables = {
host => $target_short_host_name,
network => $network_name,
ip_address => $target_ip_address,
source_file => $file,
target_directory => $target_directory,
size => $anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}),
link_speed => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}),
eta_copy_time => $say_copy_time,
};
$anvil->data->{sys}{progress} += 2;
$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
$anvil->Job->update_progress({
progress => $anvil->data->{sys}{progress},
message => "message_0195",
variables => $variables,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => $variables});
my $problem = $anvil->Storage->rsync({
debug => 2,
source => $file,
destination => "root\@".$target_ip_address.":".$target_directory,
try_again => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if (not $problem)
{
$copied = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }});
$anvil->data->{sys}{progress} += 5;
$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
$anvil->Job->update_progress({
progress => $anvil->data->{sys}{progress},
message => "message_0310",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0310"});
}
}
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }});
if (not $copied)
{
# Failed to connect, register a job instead.
my $variables = { host => $target_host_name };
$anvil->data->{sys}{progress} += 5;
$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
$anvil->Job->update_progress({
progress => $anvil->data->{sys}{progress},
message => "message_0196",
variables => $variables,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => $variables});
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE,
line => __LINE__,
job_command => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches,
job_data => "file_uuid=".$file_uuid,
job_name => "storage::pull_file",
job_title => "job_0132",
job_description => "job_0133",
job_progress => 0,
job_host_uuid => $target_host_uuid,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
}
# Mark the file as being on this host
my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({
debug => 2,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => 1,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
}
}
$anvil->Storage->push_file({
debug => 2,
file => $file,
file_uuid => $file_uuid,
});
# Call track_files, it'll make sure the file_locations are setup.
$anvil->Database->track_files({debug => 2});
# Done!
$anvil->Job->update_progress({

Write Preview
Loading…
Cancel
Save