From 254f7ef4e2950057e10b2f8278b0fa6e080b3310 Mon Sep 17 00:00:00 2001 From: digimer Date: Wed, 22 Feb 2023 02:13:19 -0500 Subject: [PATCH] This should fix the tracking of what files belong where, using the new DR links system. It also should finish (though testing is still needed) the serial rsync issue. * Created Database->track_files() as a dedicated method as trying to verify the existence of file_locations during Database->load_anvils() was fragile and prone to recursive loops. * Updated Database->insert_or_update_file_locations() to take an anvil_uuid and recursively call for each host, to maintain compatibility with the old ways, and make it simpler to add an entry for both sub-nodes in an Anvil!. * Created Storage->push_file() that takes a file and rsync's it to all other machines, or creates a job for the file to be pulled if the target can't be accessed. * Updated anvil-manage-files and anvil-sync-shared to use the new Storage->push_files and Database->track_files methods. Signed-off-by: digimer --- Anvil/Tools/Database.pm | 468 +++++++++++++++++++++++++++------------ Anvil/Tools/Storage.pm | 266 ++++++++++++++++++++++ notes | 5 + tools/anvil-manage-files | 61 +++-- tools/anvil-sync-shared | 171 +------------- 5 files changed, 645 insertions(+), 326 deletions(-) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 8bb634c2..8d2cd92e 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -97,6 +97,7 @@ my $THIS_FILE = "Database.pm"; # refresh_timestamp # resync_databases # shutdown +# track_files # update_host_status # write # _add_to_local_config @@ -401,8 +402,8 @@ sub check_file_locations # Get all the Anvil! systems we know of. $anvil->Database->get_hosts({debug => $debug}); - $anvil->Database->get_files({debug => $debug}); - $anvil->Database->get_file_locations({debug => $debug}); + #$anvil->Database->get_files({debug => $debug}); + #$anvil->Database->get_file_locations({debug => $debug}); foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{hosts}{host_name}}) { @@ -2830,135 +2831,6 @@ WHERE }}); } } - - # Track the files on this Anvil! - foreach my $file_location_uuid (keys %{$anvil->data->{file_locations}{file_location_uuid}}) - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); - - # If either node is set have this file, make sure both are. - my $file_uuid = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid}; - my $anvil_needs_file = 0; - $anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid} = 0; - $anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid} = 0; - if ((exists $anvil->data->{file_locations}{host_uuid}{$anvil_node1_host_uuid}{file_uuid}{$file_uuid}) && - ($anvil->data->{file_locations}{host_uuid}{$anvil_node1_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid})) - { - my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$anvil_node1_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; - my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - file_location_uuid => $file_location_uuid, - file_location_active => $file_location_active, - }}); - - if (not $file_location_active) - { - $anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid} = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "needs_file::${file_uuid}::${anvil_node1_host_uuid}" => $anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid}, - }}); - } - } - if ((exists $anvil->data->{file_locations}{host_uuid}{$anvil_node2_host_uuid}{file_uuid}{$file_uuid}) && - ($anvil->data->{file_locations}{host_uuid}{$anvil_node2_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid})) - { - my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$anvil_node2_host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; - my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - file_location_uuid => $file_location_uuid, - file_location_active => $file_location_active, - }}); - - if (not $file_location_active) - { - $anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid} = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "needs_file::${file_uuid}::${anvil_node2_host_uuid}" => $anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid}, - }}); - } - } - - # If either node has the file, both nodes (and all DRs) need the file. - if (($anvil->{needs_file}{$file_uuid}{$anvil_node1_host_uuid}) or ($anvil->{needs_file}{$file_uuid}{$anvil_node2_host_uuid})) - { - $anvil_needs_file = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_needs_file => $anvil_needs_file }}); - } - - foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid) - { - # If the Anvil! needs the file, make sure that both nodes and any connected DRs have it as well. - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - host_uuid => $host_uuid, - }}); - if ($anvil_needs_file) - { - if (not $anvil->{needs_file}{$file_uuid}{$host_uuid}) - { - $anvil->Database->insert_or_update_file_locations({ - debug => $debug, - file_location_file_uuid => $file_location_uuid, - file_location_host_uuid => $host_uuid, - file_location_active => 1, - }); - } - } - } - - # If the Anvil! node needs the file, record it as being on this Anvil! node. - if ($anvil_needs_file) - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "file_locations::file_location_uuid::${file_location_uuid}::file_location_host_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid}, - "file_locations::file_location_uuid::${file_location_uuid}::file_location_active" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}, - }}); - next if not exists $anvil->data->{files}{file_uuid}{$file_uuid}; - - my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - file_uuid => $file_uuid, - file_name => $file_name, - }}); - - # If the file was deleted, this won't exist - next if not exists $anvil->data->{files}{file_uuid}{$file_uuid}; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name} = $file_name; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory}; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_size}; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum}; - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_size" => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}}).")", - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum}, - "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type}, - }}); - - # Make it so that we can list the files by name. - $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_uuid; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, - }}); - - # Make sure linked DR hosts have this file, also. - foreach my $host_uuid (keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}}) - { - my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; - my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; - - if (not $file_location_active) - { - $anvil->Database->insert_or_update_file_locations({ - debug => $debug, - file_location_file_uuid => $file_location_uuid, - file_location_host_uuid => $host_uuid, - file_location_active => 1, - }); - } - } - } - } } return(0); @@ -3372,6 +3244,18 @@ FROM "file_locations::file_location_uuid::${file_location_uuid}::modified_date" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date}, }}); + ### TODO: Remove this when the WebUI is updated. + # If this host is a node in an Anvil!, set the old 'file_location_anvil_uuid' to maintain + # backwards compatibility. + if ((exists $anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}) && + ($anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}{anvil_uuid})) + { + $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_anvil_uuid} = $anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}{anvil_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "file_locations::file_location_uuid::${file_location_uuid}::file_location_anvil_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_anvil_uuid}, + }}); + } + # Make it easy to find files by anvil and file UUID. $anvil->data->{file_locations}{host_uuid}{$file_location_host_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid} = $file_location_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -8002,6 +7886,12 @@ Parameters; If not passed, a check will be made to see if an existing entry is found for C<< file_location_file_uuid >>. If found, that entry will be updated. If not found, a new record will be inserted. +=head3 file_location_anvil_uuid (required) + +This is the C<< anvils >> -> C<< anvil_uuid >> being referenced. This works by figuring out which hosts are a member of the Anvil! node, and which DR hosts are linked, and makes a recursive call to this method for each of their C<< hosts >> -> C<< host_uuid >>. + +B<< Note >>: When this is used, a comma-separated list of C<< host_uuid=file_location_uuid >> is returned. + =head3 file_location_file_uuid (required) This is the C<< files >> -> C<< file_uuid >> being referenced. @@ -8025,21 +7915,23 @@ sub insert_or_update_file_locations my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }}); - my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : ""; - my $file = defined $parameter->{file} ? $parameter->{file} : ""; - my $line = defined $parameter->{line} ? $parameter->{line} : ""; - my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : ""; - my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : ""; - my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : ""; - my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : ""; + my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : ""; + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $line = defined $parameter->{line} ? $parameter->{line} : ""; + my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : ""; + my $file_location_anvil_uuid = defined $parameter->{file_location_anvil_uuid} ? $parameter->{file_location_anvil_uuid} : ""; + my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : ""; + my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : ""; + my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - uuid => $uuid, - file => $file, - line => $line, - file_location_uuid => $file_location_uuid, - file_location_file_uuid => $file_location_file_uuid, - file_location_host_uuid => $file_location_host_uuid, - file_location_active => $file_location_active, + uuid => $uuid, + file => $file, + line => $line, + file_location_uuid => $file_location_uuid, + file_location_anvil_uuid => $file_location_anvil_uuid, + file_location_file_uuid => $file_location_file_uuid, + file_location_host_uuid => $file_location_host_uuid, + file_location_active => $file_location_active, }}); if (not $file_location_file_uuid) @@ -8061,6 +7953,59 @@ sub insert_or_update_file_locations return(""); } + # If we've got an Anvil! uuid, find out the hosts and DR links connected to the Anvil! are found and + # this method is recursively called for each host. + if ($file_location_anvil_uuid) + { + $anvil->Database->get_anvils({debug => $debug}); + if (not exists $anvil->data->{anvils}{anvil_uuid}{$file_location_anvil_uuid}) + { + # Bad Anvil! UUID. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0169", variables => { anvil_uuid => $file_location_anvil_uuid }}); + return(""); + } + my $hosts = (); + push @{$hosts}, $anvil->data->{anvils}{anvil_uuid}{$file_location_anvil_uuid}{anvil_node1_host_uuid}; + push @{$hosts}, $anvil->data->{anvils}{anvil_uuid}{$file_location_anvil_uuid}{anvil_node2_host_uuid}; + if (exists $anvil->data->{dr_links}{by_anvil_uuid}{$file_location_anvil_uuid}) + { + foreach my $dr_link_host_uuid (sort {$a cmp $b} keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$file_location_anvil_uuid}{dr_link_host_uuid}}) + { + my $dr_link_uuid = $anvil->data->{dr_links}{by_anvil_uuid}{$file_location_anvil_uuid}{dr_link_host_uuid}{$dr_link_host_uuid}{dr_link_uuid}; + my $dr_link_note = $anvil->data->{dr_links}{dr_link_uuid}{$dr_link_uuid}{dr_link_note}; + my $dr_link_short_host_name = $anvil->data->{hosts}{host_uuid}{$dr_link_host_uuid}{short_host_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "s1:dr_link_host_uuid" => $dr_link_host_uuid, + "s2:dr_link_uuid" => $dr_link_uuid, + "s3:dr_link_note" => $dr_link_note, + "s4:dr_link_short_host_name" => $dr_link_short_host_name, + }}); + + next if $dr_link_note eq "DELETED"; + push @{$hosts}, $dr_link_host_uuid; + } + } + my $file_location_uuids = ""; + foreach my $host_uuid (@{$hosts}) + { + my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_file_uuid => $file_location_file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => $file_location_active, + }); + $file_location_uuids .= $host_uuid."=".$file_location_uuid.","; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file_location_uuid => $file_location_uuid, + file_location_uuids => $file_location_uuids, + }}); + } + $file_location_uuids =~ s/,$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuids => $file_location_uuids }}); + + return($file_location_uuids); + } + # If we don't have a UUID, see if we can find one for the given md5sum. if (not $file_location_uuid) { @@ -8124,7 +8069,7 @@ INSERT INTO SELECT file_location_file_uuid, file_location_host_uuid, - file_location_active, + file_location_active FROM file_locations WHERE @@ -17575,6 +17520,237 @@ AND } +=head2 track_file + +This looks at all files in the database, and then for all Anvil! systems and linked DR hosts, ensures that there's a corresponding C<< file_locations >> entry. + +This method takes no parameters. + +=cut +sub track_files +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->track_files()" }}); + + my $anvils = keys %{$anvil->data->{anvils}{anvil_name}}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvils => $anvils }}); + if (not $anvils) + { + $anvil->Database->get_anvils({debug => $debug}); + } + + my $files = keys %{$anvil->data->{files}{file_uuid}}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { files => $files }}); + if (not $files) + { + $anvil->Database->get_files({debug => $debug}); + } + + my $file_locations = keys %{$anvil->data->{file_locations}{file_location_uuid}}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_locations => $file_locations }}); + if (not $file_locations) + { + $anvil->Database->get_file_locations({debug => $debug}); + } + + my $dr_link_uuid = keys %{$anvil->data->{dr_links}{dr_link_uuid}}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { dr_link_uuid => $dr_link_uuid }}); + if (not $dr_link_uuid) + { + $anvil->Database->get_dr_links({debug => $debug}); + } + + foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}}) + { + my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid}; + my $anvil_description = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_description}; + my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; + my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:anvil_name' => $anvil_name, + 's2:anvil_uuid' => $anvil_uuid, + 's3:anvil_description' => $anvil_description, + 's4:anvil_node1_host_uuid' => $anvil_node1_host_uuid, + 's5:anvil_node2_host_uuid' => $anvil_node2_host_uuid, + }}); + + # Loop through all files and see if there's a corresponding file_location for each sub-node. + my $reload = 0; + foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}}) + { + my $file_uuid = $anvil->data->{files}{file_name}{$file_name}{file_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:file_name' => $file_name, + 's2:file_uuid' => $file_uuid, + }}); + + foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }}); + + if ((not exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}) or + ($anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid})) + { + # Add it + $reload = 1; + my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_file_uuid => $file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + reload => $reload, + file_location_uuid => $file_location_uuid, + }}); + } + } + } + if ($reload) + { + $anvil->Database->get_file_locations({debug => $debug}); + } + + # Track the files on this Anvil! + foreach my $file_location_uuid (keys %{$anvil->data->{file_locations}{file_location_uuid}}) + { + my $file_uuid = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid}; + my $file_type = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type}; + my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file_location_uuid => $file_location_uuid, + file_uuid => $file_uuid, + file_type => $file_type, + file_name => $file_name, + }}); + next if $file_type eq "DELETED"; + + ### TODO - Left off here, not adding DR links. + my $anvil_needs_file = 0; + foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }}); + + if ((exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}) && + ($anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid})) + { + my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file_location_uuid => $file_location_uuid, + file_location_active => $file_location_active, + }}); + + if ($file_location_active) + { + $anvil_needs_file = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_needs_file => $anvil_needs_file }}); + } + } + } + + # If either node wanted the file, both nodes and all linked DRs need it. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_needs_file => $anvil_needs_file }}); + if ($anvil_needs_file) + { + # Update the hosts + foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }}); + + my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_file_uuid => $file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); + } + + # Make sure linked DR hosts have this file, also. + foreach my $host_uuid (keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{dr_host}}) + { + my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; + + if (not $file_location_active) + { + my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_file_uuid => $file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); + } + } + + # If the file was deleted, this won't exist + next if not exists $anvil->data->{files}{file_uuid}{$file_uuid}; + + # Record that this Anvil! node has this file. + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name} = $file_name; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory}; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_size}; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum}; + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_size" => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}}).")", + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_md5sum" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum}, + "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_type" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type}, + }}); + + # Make it so that we can list the files by name. + $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, + }}); + + # Make sure linked DR hosts have this file, also. + foreach my $host_uuid (keys %{$anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}}) + { + my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + host_name => $host_name, + host_uuid => $host_uuid, + }}); + + my $file_location_uuid = ""; + my $file_location_active = 0; + if (exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}) + { + $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file_location_uuid => $file_location_uuid, + file_location_active => $file_location_active, + }}); + + if (not $file_location_active) + { + my ($file_location_uuid) = $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_file_uuid => $file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); + } + } + } + } + } + + return(0); +} + + =head2 update_host_status This is a variant on C<< insert_or_update_hosts >> designed only to update the power status of a host. diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 29553250..f2055da6 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -35,6 +35,7 @@ my $THIS_FILE = "Storage.pm"; # move_file # parse_df # parse_lsblk +# push_file # read_config # read_file # read_mode @@ -3451,6 +3452,271 @@ sub parse_lsblk } +=head2 push_file + +This takes a file and pushes it to all other machines in the cluster, serially. For machines that can't be accessed, a job is registered to pull the file. + +If C<< switches::job-uuid >> is set, the corresponding job will be updated. The progress assumes that C<< sys::progress >> is set. + +Parameters; + +=head3 file (required) + +This is the source file to copy from locally and push it to all peers' C<< /mnt/shared/files/ >> directory. + +=cut +sub push_file +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->push_file()" }}); + + # Setup default values + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file => $file, + file_uuid => $file_uuid, + }}); + + if (not $file) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Storage->push_file()", parameter => "file" }}); + return("!!error!!"); + } + if (not -f $file) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0105", variables => { file => $file }}); + return("!!error!!"); + } + + $anvil->Database->get_files({debug => $debug}); + my $file_size = 0; + my ($file_directory, $file_name) = ($file =~ /^(.*)\/(.*?)$/); + $file_directory =~ s/\/$//g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:file_directory' => $file_directory, + 's2:file_name' => $file_name, + }}); + if (not $file_uuid) + { + # Can we find the file? + foreach my $this_file_uuid (keys %{$anvil->data->{files}{file_uuid}}) + { + my $this_file_name = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_name}; + my $this_file_directory = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_directory}; + $this_file_directory =~ s/\/$//g; + my $this_file_size = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_size}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + this_file_uuid => $this_file_uuid, + this_file_directory => $this_file_directory, + this_file_name => $this_file_name, + this_file_size => $this_file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_file_size}).")", + }}); + + if (($file_name eq $this_file_name) && + ($file_directory eq $this_file_directory)) + { + # Found it. + $file_uuid = $this_file_uuid; + $file_size = $this_file_size; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file_uuid => $file_uuid, + file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", + }}); + last; + } + } + } + + if (not $file_uuid) + { + $file_size = (stat($file))[7]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", + }}); + } + + # Now copy this to our peers. We're going to do this serially so that we don't overwhelm the system, + # Any hosts not currently online will have a job registered. + $anvil->Database->get_hosts; + my $host_uuid = $anvil->Get->host_uuid(); + my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}; + my $target_directory = $anvil->data->{path}{directories}{shared}{files}."/"; + foreach my $do_host_type ("striker", "node", "dr") + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { do_host_type => $do_host_type }}); + foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}}) + { + my $target_host_uuid = $anvil->data->{sys}{hosts}{by_name}{$target_host_name}; + my $target_host_type = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{host_type}; + my $target_short_host_name = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{short_host_name}; + next if $target_host_uuid eq $host_uuid; + next if $target_host_type ne $do_host_type; + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:target_host_name' => $target_host_name, + 's2:target_host_uuid' => $target_host_uuid, + 's3:target_host_type' => $target_host_type, + 's4:target_short_host_name' => $target_short_host_name, + }}); + + my $matches = $anvil->Network->find_access({ + debug => 2, + target => $target_short_host_name, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }}); + next if not $matches; + next if $matches =~ /\D/; + + # Find a matching IP. + # We prefer to use least to most used networks, with the IFN being the last choice. + my $copied = 0; + foreach my $network ("mn", "bcn", "sn", "ifn") + { + next if $copied; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network => $network }}); + foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}}) + { + next if $copied; + next if $network_name !~ /^$network/i; + my $local_interface = $anvil->data->{network_access}{$network_name}{local_interface}; + my $local_speed = $anvil->data->{network_access}{$network_name}{local_speed}; + my $local_ip_address = $anvil->data->{network_access}{$network_name}{local_ip_address}; + my $local_subnet_mask = $anvil->data->{network_access}{$network_name}{local_subnet_mask}; + my $target_interface = $anvil->data->{network_access}{$network_name}{target_interface}; + my $target_speed = $anvil->data->{network_access}{$network_name}{target_speed}; + my $target_ip_address = $anvil->data->{network_access}{$network_name}{target_ip_address}; + my $target_subnet_mask = $anvil->data->{network_access}{$network_name}{target_subnet_mask}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:local_interface' => $local_interface, + 's2:local_speed' => $local_speed, + 's3:local_ip_address' => $local_ip_address, + 's4:local_subnet_mask' => $local_subnet_mask, + 's5:target_interface' => $target_interface, + 's6:target_speed' => $target_speed, + 's7:target_ip_address' => $target_ip_address, + 's8:target_subnet_mask' => $target_subnet_mask, + }}); + + my $access = $anvil->Remote->test_access({target => $target_ip_address}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); + + if ($access) + { + ### Rsync! + # Estimate how long this will take. First, get the speed in + # Mbps and turn it into bytes per second, then into bytes per + # second. We'll take 10% off, then calculate how many seconds + # the copy will take. + my $link_mbps = $target_speed > $local_speed ? $target_speed : $local_speed; + my $link_bps = $link_mbps * 1000000; + my $link_bytes_sec = int($link_bps / 8); + my $adjusted_byptes_sec = int($link_bytes_sec * 0.9); + my $copy_seconds = int($file_size / $adjusted_byptes_sec); + my $say_copy_time = $anvil->Convert->time({'time' => $copy_seconds, translate => 1}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:link_mbps' => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}), + 's2:link_bps' => $anvil->Convert->add_commas({number => $link_bps})." ".$anvil->Words->string({string => "#!string!suffix_0048!#"}), + 's3:link_bytes_sec' => $anvil->Convert->add_commas({number => $link_bytes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}), + 's4:adjusted_byptes_sec' => $anvil->Convert->add_commas({number => $adjusted_byptes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}), + 's5:copy_seconds' => $anvil->Convert->add_commas({number => $copy_seconds})." ".$anvil->Words->string({string => "#!string!suffix_0007!#"}), + 's6:say_copy_time' => $say_copy_time, + }}); + + my $variables = { + host => $target_short_host_name, + network => $network_name, + ip_address => $target_ip_address, + source_file => $file, + target_directory => $target_directory, + size => $anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}), + link_speed => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}), + eta_copy_time => $say_copy_time, + }; + $anvil->data->{sys}{progress} += 2; + $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0195", + variables => $variables, + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => $variables}); + my $problem = $anvil->Storage->rsync({ + debug => 2, + source => $file, + destination => "root\@".$target_ip_address.":".$target_directory, + try_again => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if (not $problem) + { + $copied = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }}); + + $anvil->data->{sys}{progress} += 5; + $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0310", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0310"}); + } + } + } + } + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); + if ($file_uuid) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }}); + if (not $copied) + { + # Failed to connect, register a job instead. + my $variables = { host => $target_host_name }; + $anvil->data->{sys}{progress} += 5; + $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0196", + variables => $variables, + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => $variables}); + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + file => $THIS_FILE, + line => __LINE__, + job_command => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, + job_data => "file_uuid=".$file_uuid, + job_name => "storage::pull_file", + job_title => "job_0132", + job_description => "job_0133", + job_progress => 0, + job_host_uuid => $target_host_uuid, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + } + + # Mark the file as being on this host + if ($do_host_type ne "striker") + { + my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({ + debug => 2, + file_location_file_uuid => $file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); + } + } + } + } + + return(0); +} + + =head2 read_config This method is used to read 'Anvil::Tools' style configuration files. These configuration files are in the format: diff --git a/notes b/notes index 5aa2fe5a..c4012abd 100644 --- a/notes +++ b/notes @@ -1,4 +1,9 @@ +Common queries; +* SELECT a.job_uuid, b.host_name, a.job_command, a.job_data, a.job_progress, a.job_status FROM jobs a, hosts b WHERE a.job_host_uuid = b.host_uuid AND a.job_progress != 100; +* SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC; +* SELECT a.dr_link_uuid, b.host_name, c.anvil_name, a.dr_link_note FROM dr_links a, hosts b, anvils c WHERE a.dr_link_host_uuid = b.host_uuid AND a.dr_link_anvil_uuid = c.anvil_uuid ORDER BY c.anvil_name ASC, b.host_name ASC; + When pairing Striker, make sure new config goes to all known nodes! diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index 7ec94178..1145ebed 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -564,7 +564,8 @@ sub check_incoming }}); # Calculate the md5sum? - my $file_md5sum = $recorded_md5sum; + my $added_to_db = 0; + my $file_md5sum = $recorded_md5sum; if ((not $file_uuid) or ($file_size != $recorded_size)) { # It's possible the file is still uploading, so sleep for a bit and see if the @@ -608,7 +609,7 @@ sub check_incoming # Insert or update the files entry. ($file_uuid) = $anvil->Database->insert_or_update_files({ - debug => 3, + debug => 2, file_uuid => $file_uuid, file_name => $file_name, file_directory => $file_directory, @@ -617,15 +618,21 @@ sub check_incoming file_mtime => $file_mtime, file_type => $say_mimetype, }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); + + if ($file_uuid) + { + $added_to_db = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { added_to_db => $added_to_db }}); + } } # If we still don't have a file UUID for some reason, skip this file. - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); next if not $file_uuid; # Are we in the incoming directory? If so, move the file. - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path, incoming_directory => $incoming_directory, }}); @@ -640,28 +647,34 @@ sub check_incoming my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files}; $target .= "/"; $target =~ s/\/\//\//g; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { target => $target }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => { file => $full_path, target => $target, }}); - # Wait in case it's still being uploaded. - $anvil->Storage->_wait_if_changing({ - file => $full_path, - last_size => $file_size, - }); - - $anvil->Storage->move_file({ - debug => 3, + my $problem = $anvil->Storage->move_file({ + debug => 2, source_file => $full_path, target_file => $target, }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); + if (not $problem) + { + $full_path = $target."/".$file_name; + $file_directory = $target; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + file_directory => $file_directory, + full_path => $full_path, + }}); + } # Update the file_directory. - ($file_uuid) = $anvil->Database->insert_or_update_files({ - debug => 3, + $target =~ s/\/$//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target => $target }}); + ($file_uuid) = $anvil->Database->insert_or_update_files({ + debug => 2, file_uuid => $file_uuid, file_name => $file_name, file_directory => $target, @@ -670,7 +683,20 @@ sub check_incoming file_mtime => $file_mtime, file_type => $say_mimetype, }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); + } + + if ($added_to_db) + { + # Push it to all other hosts now. + $anvil->Storage->push_file({ + debug => 2, + file => $full_path, + file_uuid => $file_uuid, + }); + + # Call track_files, it'll make sure the file_locations are setup. + $anvil->Database->track_files({debug => 2}); } } @@ -729,6 +755,7 @@ WHERE { $query .= " file_name = ".$anvil->Database->quote($file_name)."\n"; } + $query .= ";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; diff --git a/tools/anvil-sync-shared b/tools/anvil-sync-shared index 32ce105f..fd1f72fe 100755 --- a/tools/anvil-sync-shared +++ b/tools/anvil-sync-shared @@ -574,169 +574,14 @@ sub process_incoming_file # Now copy this to our peers. We're going to do this serially so that we don't overwhelm the system, # Any hosts not currently online will have a job registered. - $anvil->Database->get_hosts; - my $host_uuid = $anvil->Get->host_uuid(); - my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}; - my $target_directory = $anvil->data->{path}{directories}{shared}{files}."/"; - foreach my $do_host_type ("striker", "node", "dr") - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { do_host_type => $do_host_type }}); - foreach my $target_host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}}) - { - my $target_host_uuid = $anvil->data->{sys}{hosts}{by_name}{$target_host_name}; - my $target_host_type = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{host_type}; - my $target_short_host_name = $anvil->data->{hosts}{host_uuid}{$target_host_uuid}{short_host_name}; - next if $target_host_uuid eq $host_uuid; - next if $target_host_type ne $do_host_type; - - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:target_host_name' => $target_host_name, - 's2:target_host_uuid' => $target_host_uuid, - 's3:target_host_type' => $target_host_type, - 's4:target_short_host_name' => $target_short_host_name, - }}); - - my $matches = $anvil->Network->find_access({ - debug => 2, - target => $target_short_host_name, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }}); - next if not $matches; - next if $matches =~ /\D/; - - # Find a matching IP. - # We prefer to use least to most used networks, with the IFN being the last choice. - my $copied = 0; - foreach my $network ("mn", "bcn", "sn", "ifn") - { - next if $copied; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network => $network }}); - foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}}) - { - next if $copied; - next if $network_name !~ /^$network/i; - my $local_interface = $anvil->data->{network_access}{$network_name}{local_interface}; - my $local_speed = $anvil->data->{network_access}{$network_name}{local_speed}; - my $local_ip_address = $anvil->data->{network_access}{$network_name}{local_ip_address}; - my $local_subnet_mask = $anvil->data->{network_access}{$network_name}{local_subnet_mask}; - my $target_interface = $anvil->data->{network_access}{$network_name}{target_interface}; - my $target_speed = $anvil->data->{network_access}{$network_name}{target_speed}; - my $target_ip_address = $anvil->data->{network_access}{$network_name}{target_ip_address}; - my $target_subnet_mask = $anvil->data->{network_access}{$network_name}{target_subnet_mask}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:local_interface' => $local_interface, - 's2:local_speed' => $local_speed, - 's3:local_ip_address' => $local_ip_address, - 's4:local_subnet_mask' => $local_subnet_mask, - 's5:target_interface' => $target_interface, - 's6:target_speed' => $target_speed, - 's7:target_ip_address' => $target_ip_address, - 's8:target_subnet_mask' => $target_subnet_mask, - }}); - - my $access = $anvil->Remote->test_access({target => $target_ip_address}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }}); - - if ($access) - { - ### Rsync! - # Estimate how long this will take. First, get the speed in - # Mbps and turn it into bytes per second, then into bytes per - # second. We'll take 10% off, then calculate how many seconds - # the copy will take. - my $link_mbps = $target_speed > $local_speed ? $target_speed : $local_speed; - my $link_bps = $link_mbps * 1000000; - my $link_bytes_sec = int($link_bps / 8); - my $adjusted_byptes_sec = int($link_bytes_sec * 0.9); - my $copy_seconds = int($file_size / $adjusted_byptes_sec); - my $say_copy_time = $anvil->Convert->time({'time' => $copy_seconds, translate => 1}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - 's1:link_mbps' => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}), - 's2:link_bps' => $anvil->Convert->add_commas({number => $link_bps})." ".$anvil->Words->string({string => "#!string!suffix_0048!#"}), - 's3:link_bytes_sec' => $anvil->Convert->add_commas({number => $link_bytes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}), - 's4:adjusted_byptes_sec' => $anvil->Convert->add_commas({number => $adjusted_byptes_sec})." ".$anvil->Words->string({string => "#!string!suffix_0060!#"}), - 's5:copy_seconds' => $anvil->Convert->add_commas({number => $copy_seconds})." ".$anvil->Words->string({string => "#!string!suffix_0007!#"}), - 's6:say_copy_time' => $say_copy_time, - }}); - - my $variables = { - host => $target_short_host_name, - network => $network_name, - ip_address => $target_ip_address, - source_file => $file, - target_directory => $target_directory, - size => $anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}), - link_speed => $anvil->Convert->add_commas({number => $link_mbps})." ".$anvil->Words->string({string => "#!string!suffix_0050!#"}), - eta_copy_time => $say_copy_time, - }; - $anvil->data->{sys}{progress} += 2; - $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; - $anvil->Job->update_progress({ - progress => $anvil->data->{sys}{progress}, - message => "message_0195", - variables => $variables, - }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => $variables}); - my $problem = $anvil->Storage->rsync({ - debug => 2, - source => $file, - destination => "root\@".$target_ip_address.":".$target_directory, - try_again => 1, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); - if (not $problem) - { - $copied = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }}); - - $anvil->data->{sys}{progress} += 5; - $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; - $anvil->Job->update_progress({ - progress => $anvil->data->{sys}{progress}, - message => "message_0310", - }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0310"}); - } - } - } - } - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { copied => $copied }}); - if (not $copied) - { - # Failed to connect, register a job instead. - my $variables = { host => $target_host_name }; - $anvil->data->{sys}{progress} += 5; - $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; - $anvil->Job->update_progress({ - progress => $anvil->data->{sys}{progress}, - message => "message_0196", - variables => $variables, - }); - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => $variables}); - my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ - file => $THIS_FILE, - line => __LINE__, - job_command => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, - job_data => "file_uuid=".$file_uuid, - job_name => "storage::pull_file", - job_title => "job_0132", - job_description => "job_0133", - job_progress => 0, - job_host_uuid => $target_host_uuid, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); - } - - # Mark the file as being on this host - my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({ - debug => 2, - file_location_file_uuid => $file_uuid, - file_location_host_uuid => $host_uuid, - file_location_active => 1, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); - } - } + $anvil->Storage->push_file({ + debug => 2, + file => $file, + file_uuid => $file_uuid, + }); + + # Call track_files, it'll make sure the file_locations are setup. + $anvil->Database->track_files({debug => 2}); # Done! $anvil->Job->update_progress({