Merge pull request #238 from ClusterLabs/anvil-tools-dev

Anvil tools dev
main
Digimer 2 years ago committed by GitHub
commit 069fab6a69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      Anvil/Tools.pm
  2. 18
      Anvil/Tools/Database.pm
  3. 16
      Anvil/Tools/Get.pm
  4. 14
      Anvil/Tools/Log.pm
  5. 117
      Anvil/Tools/Network.pm
  6. 9
      Anvil/Tools/Remote.pm
  7. 102
      Anvil/Tools/Storage.pm
  8. 63
      Anvil/Tools/Words.pm
  9. 38
      man/anvil-manage-files.8
  10. 12
      scancore-agents/scan-drbd/scan-drbd
  11. 4
      scancore-agents/scan-drbd/scan-drbd.xml
  12. 13
      scancore-agents/scan-ipmitool/scan-ipmitool
  13. 3
      scancore-agents/scan-ipmitool/scan-ipmitool.xml
  14. 88
      scancore-agents/scan-lvm/scan-lvm
  15. 3
      scancore-agents/scan-lvm/scan-lvm.xml
  16. 23
      scancore-agents/scan-network/scan-network
  17. 2
      scancore-agents/scan-network/scan-network.xml
  18. 10
      share/words.xml
  19. 90
      tools/anvil-daemon
  20. 619
      tools/anvil-manage-files
  21. 14
      tools/anvil-provision-server
  22. 72
      tools/anvil-sync-shared
  23. 14
      tools/anvil-update-states
  24. 1
      tools/striker-access-database

@ -415,6 +415,7 @@ sub nice_exit
$anvil->data->{HANDLE}{'log'}{main} = ""; $anvil->data->{HANDLE}{'log'}{main} = "";
} }
#print "Exiting with RC: [".$exit_code."]\n";
exit($exit_code); exit($exit_code);
} }
@ -772,7 +773,11 @@ sub _get_hash_reference
my $parameter = shift; my $parameter = shift;
my $anvil = $self; my $anvil = $self;
die "$THIS_FILE ".__LINE__."; The hash key string: [".$parameter->{key}."] doesn't seem to be valid. It should be a string in the format 'foo::bar::baz'.\n" if $parameter->{key} !~ /::/; if ($parameter->{key} !~ /::/)
{
print "$THIS_FILE ".__LINE__."; The hash key string: [".$parameter->{key}."] doesn't seem to be valid. It should be a string in the format 'foo::bar::baz'.\n";
$anvil->nice_exit({exit_code => 1});
}
# Split up the keys. # Split up the keys.
my $key = $parameter->{key} ? $parameter->{key} : ""; my $key = $parameter->{key} ? $parameter->{key} : "";
@ -1111,6 +1116,7 @@ sub _set_paths
'anvil-join-anvil' => "/usr/sbin/anvil-join-anvil", 'anvil-join-anvil' => "/usr/sbin/anvil-join-anvil",
'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode", 'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode",
'anvil-manage-dr' => "/usr/sbin/anvil-manage-dr", 'anvil-manage-dr' => "/usr/sbin/anvil-manage-dr",
'anvil-manage-files' => "/usr/sbin/anvil-manage-files",
'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall", 'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall",
'anvil-manage-keys' => "/usr/sbin/anvil-manage-keys", 'anvil-manage-keys' => "/usr/sbin/anvil-manage-keys",
'anvil-manage-power' => "/usr/sbin/anvil-manage-power", 'anvil-manage-power' => "/usr/sbin/anvil-manage-power",

@ -381,7 +381,7 @@ sub backup_database
=head2 check_file_locations =head2 check_file_locations
This method checks to see that there is a corresponding entry in C<< file_locations >> for all Anvil! systems and files in the database. Any that are found to be missing will be set to C<< file_location_active >> -> c<< false >>. This method checks to see that there is a corresponding entry in C<< file_locations >> for all Anvil! systems and files in the database. Any that are found to be missing will be set to C<< file_location_active >> -> c<< true >>.
This method takes no parameters. This method takes no parameters.
@ -420,7 +420,7 @@ sub check_file_locations
debug => $debug, debug => $debug,
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_anvil_uuid => $anvil_uuid, file_location_anvil_uuid => $anvil_uuid,
file_location_active => 0, file_location_active => 1,
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
} }
@ -1644,7 +1644,6 @@ sub connect
query => $query, query => $query,
}}); }});
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
die;
} }
# Check to see if the schema needs to be loaded. # Check to see if the schema needs to be loaded.
@ -2711,7 +2710,7 @@ WHERE
next if not exists $anvil->data->{files}{file_uuid}{$file_uuid}; next if not exists $anvil->data->{files}{file_uuid}{$file_uuid};
my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name}; my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_name => $file_name, file_name => $file_name,
}}); }});
@ -2723,7 +2722,7 @@ WHERE
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_size}; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_size};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum}; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_md5sum} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum};
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type}; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_type} = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name}, "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_name" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_name},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory}, "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_directory},
"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_size" => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}}).")", "anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_uuid}::file_size" => $anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}})." (".$anvil->Convert->add_commas({number => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_size}}).")",
@ -2733,7 +2732,7 @@ WHERE
# Make it so that we can list the files by name. # Make it so that we can list the files by name.
$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_uuid; $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid} = $file_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, "anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid},
}}); }});
} }
@ -15379,10 +15378,13 @@ sub mark_active
return(0); return(0);
} }
my $caller = $ENV{_} ? $ENV{_} : "unknown";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'caller' => $caller }});
# Record that we're using each available striker DB UUID. # Record that we're using each available striker DB UUID.
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}})
{ {
my $state_name = "db_in_use::".$uuid."::".$$; my $state_name = "db_in_use::".$uuid."::".$$."::".$caller;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
set => $set, set => $set,
state_name => $state_name, state_name => $state_name,
@ -16444,7 +16446,6 @@ sub resync_databases
{ {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0114", variables => { query => $query }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0114", variables => { query => $query }});
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
die;
} }
# If we don't have a row uuid, something has also gone wrong... # If we don't have a row uuid, something has also gone wrong...
@ -16455,7 +16456,6 @@ sub resync_databases
query => $query, query => $query,
}}); }});
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
die;
} }
# Record this in the unified and local hashes. # Record this in the unified and local hashes.

@ -791,11 +791,11 @@ sub bridges
if (not $test) if (not $test)
{ {
# JSON parse failed. # JSON parse failed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "error_0140", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "error_0140", variables => {
json => $output, json => $output,
error => $@, error => $@,
}}); }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0519"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0519"});
# NOTE: This is not design to be normally used. It was created as a stop-gap while waiting # NOTE: This is not design to be normally used. It was created as a stop-gap while waiting
# for resolution on: https://bugzilla.redhat.com/show_bug.cgi?id=1868467 # for resolution on: https://bugzilla.redhat.com/show_bug.cgi?id=1868467
@ -1027,7 +1027,7 @@ sub cgi
if (not $cgi->upload('upload_file')) if (not $cgi->upload('upload_file'))
{ {
# Empty file passed, looks like the user forgot to select a file to upload. # Empty file passed, looks like the user forgot to select a file to upload.
$anvil->Log->entry({log_level => 2, message_key => "log_0242", file => $THIS_FILE, line => __LINE__}); $anvil->Log->entry({level => 2, message_key => "log_0242", file => $THIS_FILE, line => __LINE__});
} }
else else
{ {
@ -1183,11 +1183,13 @@ sub date_and_time
# Are things sane? # Are things sane?
if ($use_time =~ /D/) if ($use_time =~ /D/)
{ {
die "Get->date_and_time() was called with 'use_time' set to: [$use_time]. Only a unix timestamp is allowed.\n"; warn "Get->date_and_time() was called with 'use_time' set to: [$use_time]. Only a unix timestamp is allowed.\n";
$anvil->nice_exit({exit_code => 1});
} }
if ($offset =~ /D/) if ($offset =~ /D/)
{ {
die "Get->date_and_time() was called with 'offset' set to: [$offset]. Only real number is allowed.\n"; warn "Get->date_and_time() was called with 'offset' set to: [$offset]. Only real number is allowed.\n";
$anvil->nice_exit({exit_code => 1});
} }
# Do my initial calculation. # Do my initial calculation.
@ -2297,12 +2299,14 @@ sub switches
next if $set_switch eq "?"; next if $set_switch eq "?";
next if $set_switch eq "h"; next if $set_switch eq "h";
next if $set_switch eq "help"; next if $set_switch eq "help";
next if $set_switch eq "log-secure";
next if $set_switch eq "log-db-transactions";
next if $set_switch eq "raw"; next if $set_switch eq "raw";
next if $set_switch eq "resync-db";
next if $set_switch eq "v"; next if $set_switch eq "v";
next if $set_switch eq "vv"; next if $set_switch eq "vv";
next if $set_switch eq "vvv"; next if $set_switch eq "vvv";
next if $set_switch eq "vvvv"; next if $set_switch eq "vvvv";
next if $set_switch eq "log-secure";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set_switch => $set_switch }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { set_switch => $set_switch }});
my $found = 0; my $found = 0;

@ -481,7 +481,7 @@ sub entry
my $shell_call = $log_file; my $shell_call = $log_file;
print $THIS_FILE." ".__LINE__."; shell_call: [".$shell_call."]\n" if $test; print $THIS_FILE." ".__LINE__."; shell_call: [".$shell_call."]\n" if $test;
# NOTE: Don't call '$anvil->Log->entry()' here, it will cause a loop! # NOTE: Don't call '$anvil->Log->entry()' here, it will cause a loop!
open (my $file_handle, ">>", $shell_call) or die "Failed to open: [$shell_call] for writing. The error was: $!\n"; open (my $file_handle, ">>", $shell_call) or die "Failed to open: [".$shell_call."] for writing. The error was: $!\n";
$file_handle->autoflush(1); $file_handle->autoflush(1);
$anvil->data->{HANDLE}{'log'}{main} = $file_handle; $anvil->data->{HANDLE}{'log'}{main} = $file_handle;
binmode($anvil->data->{HANDLE}{'log'}{main}, ':encoding(utf-8)'); binmode($anvil->data->{HANDLE}{'log'}{main}, ':encoding(utf-8)');
@ -494,7 +494,8 @@ sub entry
if (not $anvil->data->{HANDLE}{'log'}{main}) if (not $anvil->data->{HANDLE}{'log'}{main})
{ {
# NOTE: This can't be a normal error because we can't write to the logs. # NOTE: This can't be a normal error because we can't write to the logs.
die $THIS_FILE." ".__LINE__."; log main file handle doesn't exist, but it should by now.\n"; print $THIS_FILE." ".__LINE__."; log main file handle doesn't exist, but it should by now.\n";
exit 1;
} }
# The handle has to be wrapped in a block to make 'print' happy as it doesn't like non-scalars for file handles # The handle has to be wrapped in a block to make 'print' happy as it doesn't like non-scalars for file handles
@ -533,7 +534,8 @@ sub entry
if (not $anvil->data->{HANDLE}{'log'}{alert}) if (not $anvil->data->{HANDLE}{'log'}{alert})
{ {
# NOTE: This can't be a normal error because we can't write to the logs. # NOTE: This can't be a normal error because we can't write to the logs.
die $THIS_FILE." ".__LINE__."; log alert file handle doesn't exist, but it should by now.\n"; print $THIS_FILE." ".__LINE__."; log alert file handle doesn't exist, but it should by now.\n";
exit 1;
} }
# The handle has to be wrapped in a block to make 'print' happy as it doesn't like non-scalars for file handles # The handle has to be wrapped in a block to make 'print' happy as it doesn't like non-scalars for file handles
@ -871,11 +873,13 @@ sub variables
#die if $test; #die if $test;
if (not defined $level) if (not defined $level)
{ {
die $THIS_FILE." ".__LINE__."; Log->variables() called without 'level': [".$level."] defined from: [$source : $line]\n"; warn $THIS_FILE." ".__LINE__."; Log->variables() called without 'level': [".$level."] defined from: [$source : $line]\n";
$anvil->nice_exit({exit_code => 1});
} }
elsif (not defined $anvil->Log->level) elsif (not defined $anvil->Log->level)
{ {
die $THIS_FILE." ".__LINE__."; Log->variables() called without Log->level: [".$anvil->Log->level."] defined from: [$source : $line]\n"; warn $THIS_FILE." ".__LINE__."; Log->variables() called without Log->level: [".$anvil->Log->level."] defined from: [$source : $line]\n";
$anvil->nice_exit({exit_code => 1});
} }
print "level: [$level], logging: [".$anvil->Log->level."], secure: [$secure], logging secure: [".$anvil->Log->secure."]\n" if $test; print "level: [$level], logging: [".$anvil->Log->level."], secure: [$secure], logging secure: [".$anvil->Log->secure."]\n" if $test;
if ($level > $anvil->Log->level) if ($level > $anvil->Log->level)

@ -159,7 +159,13 @@ sub bridge_info
target => $target, target => $target,
}}); }});
my $shell_call = $anvil->data->{path}{exe}{bridge}." -json -pretty link show"; my $host = $target ? $target : $anvil->Get->short_host_name();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host => $host }});
# First get the list of bridges.
my $shell_call = $anvil->data->{path}{exe}{ip}." link show type bridge";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
my $output = ""; my $output = "";
if ($anvil->Network->is_local({host => $target})) if ($anvil->Network->is_local({host => $target}))
{ {
@ -188,6 +194,72 @@ sub bridge_info
}}); }});
} }
# Find the bridge interfaces
my $bridge = "";
foreach my $line (split/\n/, $output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
if ($line =~ /^\d+:\s+(.*?):/)
{
$bridge = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { bridge => $bridge }});
$anvil->data->{bridge}{$host}{$bridge}{found} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"bridge::${host}::${bridge}::found" => $anvil->data->{bridge}{$host}{$bridge}{found},
}});
}
next if not $bridge;
if ($line =~ /mtu (\d+) /)
{
$anvil->data->{bridge}{$host}{$bridge}{mtu} = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"bridge::${host}::${bridge}::mtu" => $anvil->data->{bridge}{$host}{$bridge}{mtu},
}});
}
if ($line =~ /link\/ether (\w\w:\w\w:\w\w:\w\w:\w\w:\w\w) /)
{
$anvil->data->{bridge}{$host}{$bridge}{mac} = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"bridge::${host}::${bridge}::mac" => $anvil->data->{bridge}{$host}{$bridge}{mac},
}});
}
}
# Now use bridge to find the interfaces connected to the bridges.
$shell_call = $anvil->data->{path}{exe}{bridge}." -json -pretty link show";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
$output = "";
if ($anvil->Network->is_local({host => $target}))
{
# Local call.
($output, my $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:output' => $output,
's2:return_code' => $return_code,
}});
}
else
{
# Remote call
($output, my $error, my $return_code) = $anvil->Remote->call({
debug => $debug,
shell_call => $shell_call,
target => $target,
user => $remote_user,
password => $password,
remote_user => $remote_user,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:output' => $output,
's2:error' => $error,
's3:return_code' => $return_code,
}});
}
# Did I get usable data? # Did I get usable data?
if ($output !~ /^\[/) if ($output !~ /^\[/)
{ {
@ -201,33 +273,46 @@ sub bridge_info
foreach my $hash_ref (@{$bridge_data}) foreach my $hash_ref (@{$bridge_data})
{ {
next if not defined $hash_ref->{master}; next if not defined $hash_ref->{master};
my $bridge = $hash_ref->{master}; my $master = $hash_ref->{master}; # This can be the bond name for bond members.
my $interface = $hash_ref->{ifname}; my $interface = $hash_ref->{ifname};
my $host = $target ? $target : $anvil->Get->short_host_name();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:bridge' => $bridge, 's1:master' => $master,
's2:interface' => $interface, 's2:interface' => $interface,
's3:host' => $host,
}}); }});
if ((not exists $anvil->data->{bridge}{$host}{$bridge}) or (ref($anvil->data->{bridge}{$host}{$bridge}{interfaces}) ne "ARRAY"))
{ # If the 'master' wasn't found in the call above, the 'master' is not a bridge.
$anvil->data->{bridge}{$host}{$bridge}{interfaces} = []; next if not exists $anvil->data->{bridge}{$host}{$master};
$anvil->data->{bridge}{$host}{$bridge}{found} = 1;
# Record this interface as being connected to this bridge.
$anvil->data->{bridge}{$host}{$master}{interface}{$interface}{found} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"bridge::${host}::${bridge}::interfaces" => $anvil->data->{bridge}{$host}{$bridge}{interfaces}, "bridge::${host}::${master}::interface::${interface}::found" => $anvil->data->{bridge}{$host}{$master}{interface}{$interface}{found},
"bridge::${host}::${bridge}::found" => $anvil->data->{bridge}{$host}{$bridge}{found},
}}); }});
}
push @{$anvil->data->{bridge}{$host}{$bridge}{interfaces}}, $interface;
# Now store the rest of the data. # Now store the rest of the data.
foreach my $key (sort {$a cmp $b} keys %{$hash_ref}) foreach my $key (sort {$a cmp $b} keys %{$hash_ref})
{ {
next if $key eq "master"; next if $key eq "master";
next if $key eq "ifname"; next if $key eq "ifname";
$anvil->data->{bridge}{$host}{$bridge}{$interface}{$key} = $hash_ref->{$key}; $anvil->data->{bridge}{$host}{$master}{interface}{$interface}{$key} = $hash_ref->{$key};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"bridge::${host}::${master}::interface::${interface}::${key}" => $anvil->data->{bridge}{$host}{$master}{interface}{$interface}{$key},
}});
}
}
# Make it easy to find the bridge an interface is in.
delete $anvil->data->{interface_to_bridge} if exists $anvil->data->{interface_to_bridge};
foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{bridge}{$host}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { bridge_name => $bridge_name }});
foreach my $interface_name (sort {$a cmp $b} keys %{$anvil->data->{bridge}{$host}{$bridge_name}{interface}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
$anvil->data->{interface_to_bridge}{$interface_name} = $bridge_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"bridge::${host}::${bridge}::${interface}::${key}" => $anvil->data->{bridge}{$host}{$bridge}{$interface}{$key}, "interface_to_bridge::${interface_name}" => $anvil->data->{interface_to_bridge}{$interface_name},
}}); }});
} }
} }
@ -1185,7 +1270,7 @@ sub find_matches
source => $source, source => $source,
line => $line, line => $line,
}}); }});
die; $anvil->nice_exit({exit_code => 1});
return(""); return("");
} }
} }

@ -789,21 +789,24 @@ sub read_snmp_oid
{ {
# Um, what are we supposed to read? # Um, what are we supposed to read?
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Remote->read_snmp_oid()", parameter => "oid" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Remote->read_snmp_oid()", parameter => "oid" }});
die; $anvil->nice_exit({exit_code => 1});
return("!!error!!"); return("!!error!!");
} }
if (not $target) if (not $target)
{ {
# Who ya gonna call? No, seriously, I have no idea... # Who ya gonna call? No, seriously, I have no idea...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Remote->read_snmp_oid()", parameter => "target" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Remote->read_snmp_oid()", parameter => "target" }});
die; $anvil->nice_exit({exit_code => 1});
return("!!error!!"); return("!!error!!");
} }
if (($mib) && (not -r $mib)) if (($mib) && (not -r $mib))
{ {
# Bad MIB path # Bad MIB path
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0163", variables => { mib => $mib }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0163", variables => { mib => $mib }});
die; $anvil->nice_exit({exit_code => 1});
return("!!error!!"); return("!!error!!");
} }

@ -46,6 +46,7 @@ my $THIS_FILE = "Storage.pm";
# update_file # update_file
# write_file # write_file
# _create_rsync_wrapper # _create_rsync_wrapper
# _wait_if_changing
=pod =pod
@ -5151,6 +5152,7 @@ fi";
# Private functions # # Private functions #
############################################################################################################# #############################################################################################################
=head2 =head2
This does the actual work of creating the C<< expect >> wrapper script and returns the path to that wrapper for C<< rsync >> calls. This does the actual work of creating the C<< expect >> wrapper script and returns the path to that wrapper for C<< rsync >> calls.
@ -5227,4 +5229,104 @@ expect eof
return($wrapper_script); return($wrapper_script);
} }
=head3 _wait_if_changing
This takes a full path to a file, and watches it for at specified number of seconds to see if the size is changing. If it is, this method waits until the file size stops changing.
Parameters;
=head3 file (required)
This is the full path to the file. If the file is not found, C<< !!error!! >> is returned.
=head3 delay (optional, default '2')
This is how long to wait before checking to see if the file has changed.
=head3 last_size (optional)
If this is set, it's the first size we compare against. If not passed, the size will be checked.
=cut
sub _wait_if_changing
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->_create_rsync_wrapper()" }});
# Check my parameters.
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $delay = defined $parameter->{delay} ? $parameter->{delay} : "";
my $last_size = defined $parameter->{last_size} ? $parameter->{last_size} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file => $file,
delay => $delay,
last_size => $last_size,
}});
if (not $delay)
{
$delay = 2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }});
}
elsif (($delay =~ /\D/) or ($delay == 0))
{
$delay = 2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { delay => $delay }});
}
if (not -e $file)
{
return("!!error!!");
}
if (not $last_size)
{
$last_size = (stat($file))[7];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")",
}});
}
my $waiting = 1;
while ($waiting)
{
sleep $delay;
my $new_size = (stat($file))[7];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file => $file,
last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")",
}});
if ($new_size == $last_size)
{
# Size seems stable
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
}
else
{
# Might still be updating, wait.
my $difference = $new_size - $last_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0724", variables => {
file => $file,
old_size_bytes => $anvil->Convert->add_commas({number => $last_size}),
old_size_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}),
new_size_bytes => $anvil->Convert->add_commas({number => $new_size}),
new_size_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $new_size}),
difference_bytes => $anvil->Convert->add_commas({number => $difference}),
difference_hr => $anvil->Convert->bytes_to_human_readable({'bytes' => $difference}),
}});
$last_size = $new_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { last_size => $last_size }});
}
}
return(0);
}
1; 1;

@ -579,7 +579,7 @@ sub parse_banged_string
} }
print $THIS_FILE." ".__LINE__."; Failed to parse the pair from: [".$variable_string."]\n"; print $THIS_FILE." ".__LINE__."; Failed to parse the pair from: [".$variable_string."]\n";
print $THIS_FILE." ".__LINE__."; Was parsing message: [".$message."] from key string: [".$key_string."]\n"; print $THIS_FILE." ".__LINE__."; Was parsing message: [".$message."] from key string: [".$key_string."]\n";
die; $anvil->nice_exit({exit_code => 1});
} }
my ($variable, $value) = ($pair =~ /^!!(.*?)!(.*?)!!$/); my ($variable, $value) = ($pair =~ /^!!(.*?)!(.*?)!!$/);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
@ -958,6 +958,7 @@ sub string
# We've got a string and variables from the caller, so inject them as needed. # We've got a string and variables from the caller, so inject them as needed.
my $loops = 0; my $loops = 0;
my $limit = $anvil->data->{defaults}{limits}{string_loops} =~ /^\d+$/ ? $anvil->data->{defaults}{limits}{string_loops} : 1000; my $limit = $anvil->data->{defaults}{limits}{string_loops} =~ /^\d+$/ ? $anvil->data->{defaults}{limits}{string_loops} : 1000;
print $THIS_FILE." ".__LINE__."; limit: [".$limit."]\n" if $test;
# If the user didn't pass in any variables, then we're in trouble. # If the user didn't pass in any variables, then we're in trouble.
if (($string =~ /#!variable!(.+?)!#/s) && ((not $variables) or (ref($variables) ne "HASH"))) if (($string =~ /#!variable!(.+?)!#/s) && ((not $variables) or (ref($variables) ne "HASH")))
@ -966,17 +967,21 @@ sub string
while ($string =~ /#!variable!(.+?)!#/s) while ($string =~ /#!variable!(.+?)!#/s)
{ {
$string =~ s/#!variable!(.*?)!#/!!variable!$1!!/s; $string =~ s/#!variable!(.*?)!#/!!variable!$1!!/s;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
# Die if I've looped too many times. # Die if I've looped too many times.
$loops++; $loops++;
print $THIS_FILE." ".__LINE__."; loops: [".$loops."]\n" if $test;
if ($loops > $limit) if ($loops > $limit)
{ {
# If we're in a web environment, print the HTML header. # If we're in a web environment, print the HTML header.
print $THIS_FILE." ".__LINE__."; environment: [".$anvil->environment."]\n" if $test;
if ($anvil->environment eq "html") if ($anvil->environment eq "html")
{ {
print "Content-type: text/html; charset=utf-8\n\n"; print "Content-type: text/html; charset=utf-8\n\n";
} }
die "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n"; print "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
} }
my $error = "[ Error ] - The method Words->string() was asked to process the string: [".$string."] which has insertion variables, but nothing was passed to the 'variables' parameter."; my $error = "[ Error ] - The method Words->string() was asked to process the string: [".$string."] which has insertion variables, but nothing was passed to the 'variables' parameter.";
@ -994,24 +999,29 @@ sub string
# restore them once we're out of this loop. # restore them once we're out of this loop.
foreach my $check ($string =~ /#!([^\s]+?)!#/) foreach my $check ($string =~ /#!([^\s]+?)!#/)
{ {
print $THIS_FILE." ".__LINE__."; check: [".$check."]\n" if $test;
if (($check !~ /^data/) && if (($check !~ /^data/) &&
($check !~ /^string/) && ($check !~ /^string/) &&
($check !~ /^variable/)) ($check !~ /^variable/))
{ {
# Simply invert the '#!...!#' to '!#...#!'. # Simply invert the '#!...!#' to '!#...#!'.
$string =~ s/#!($check)!#/!#$1#!/g; $string =~ s/#!($check)!#/!#$1#!/g;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
# Die if I've looped too many times. # Die if I've looped too many times.
$loops++; $loops++;
print $THIS_FILE." ".__LINE__."; loops: [".$loops."], limit: [".$limit."]\n" if $test;
if ($loops > $limit) if ($loops > $limit)
{ {
# If we're in a web environment, print the HTML header. # If we're in a web environment, print the HTML header.
print $THIS_FILE." ".__LINE__."; environment: [".$anvil->environment."]\n" if $test;
if ($anvil->environment eq "html") if ($anvil->environment eq "html")
{ {
print "Content-type: text/html; charset=utf-8\n\n"; print "Content-type: text/html; charset=utf-8\n\n";
} }
die "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language]. Is there a bad '#!<variable>!<value>!# replacement key? Exiting.\n"; print "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language]. Is there a bad '#!<variable>!<value>!# replacement key? Exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
} }
@ -1024,26 +1034,32 @@ sub string
language => $language, language => $language,
file => $file, file => $file,
}); });
print $THIS_FILE." ".__LINE__."; string: [".$string."], key: [".$key."], this_string: [".$this_string."]\n" if $test;
if ($this_string eq "#!not_found!#") if ($this_string eq "#!not_found!#")
{ {
# The key was bad... # The key was bad...
$string =~ s/#!string!$key!#/!!e[$key]!!/; $string =~ s/#!string!$key!#/!!e[$key]!!/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
else else
{ {
$string =~ s/#!string!$key!#/$this_string/; $string =~ s/#!string!$key!#/$this_string/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
# Die if I've looped too many times. # Die if I've looped too many times.
$loops++; $loops++;
print $THIS_FILE." ".__LINE__."; loops: [".$loops."], limit: [".$limit."]\n" if $test;
if ($loops > $limit) if ($loops > $limit)
{ {
# If we're in a web environment, print the HTML header. # If we're in a web environment, print the HTML header.
print $THIS_FILE." ".__LINE__."; environment: [".$anvil->environment."]\n" if $test;
if ($anvil->environment eq "html") if ($anvil->environment eq "html")
{ {
print "Content-type: text/html; charset=utf-8\n\n"; print "Content-type: text/html; charset=utf-8\n\n";
} }
die "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n"; print "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
} }
@ -1051,17 +1067,20 @@ sub string
while ($string =~ /#!variable!(.+?)!#/s) while ($string =~ /#!variable!(.+?)!#/s)
{ {
my $variable = $1; my $variable = $1;
print $THIS_FILE." ".__LINE__."; string: [".$string."], variable: [".$variable."]\n" if $test;
# Sometimes, #!variable!*!# is used in explaining things to users. So we need # Sometimes, #!variable!*!# is used in explaining things to users. So we need
# to escape it. It will be restored later in '_restore_protected()'. # to escape it. It will be restored later in '_restore_protected()'.
if ($variable eq "*") if ($variable eq "*")
{ {
$string =~ s/#!variable!\*!#/!#variable!*#!/; $string =~ s/#!variable!\*!#/!#variable!*#!/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
next; next;
} }
if ($variable eq "") if ($variable eq "")
{ {
$string =~ s/#!variable!\*!#/!#variable!#!/; $string =~ s/#!variable!\*!#/!#variable!#!/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
next; next;
} }
@ -1070,24 +1089,29 @@ sub string
# I can't expect there to always be a defined value in the variables # I can't expect there to always be a defined value in the variables
# array at any given position so if it is blank qw blank the key. # array at any given position so if it is blank qw blank the key.
$string =~ s/#!variable!$variable!#//; $string =~ s/#!variable!$variable!#//;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
else else
{ {
my $value = $variables->{$variable}; my $value = $variables->{$variable};
chomp $value; chomp $value;
$string =~ s/#!variable!$variable!#/$value/; $string =~ s/#!variable!$variable!#/$value/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
# Die if I've looped too many times. # Die if I've looped too many times.
$loops++; $loops++;
print $THIS_FILE." ".__LINE__."; loops: [".$loops."], limit: [".$limit."]\n" if $test;
if ($loops > $limit) if ($loops > $limit)
{ {
# If we're in a web environment, print the HTML header. # If we're in a web environment, print the HTML header.
print $THIS_FILE." ".__LINE__."; environment: [".$anvil->environment."]\n" if $test;
if ($anvil->environment eq "html") if ($anvil->environment eq "html")
{ {
print "Content-type: text/html; charset=utf-8\n\n"; print "Content-type: text/html; charset=utf-8\n\n";
} }
die "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n"; print "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
} }
@ -1095,61 +1119,77 @@ sub string
while ($string =~ /#!data!(.+?)!#/) while ($string =~ /#!data!(.+?)!#/)
{ {
my $id = $1; my $id = $1;
print $THIS_FILE." ".__LINE__."; string: [".$string."], id: [".$id."]\n" if $test;
if ($id =~ /::/) if ($id =~ /::/)
{ {
# Multi-dimensional hash. # Multi-dimensional hash.
print $THIS_FILE." ".__LINE__."; multi-dimensional\n" if $test;
my $value = $anvil->_get_hash_reference({ key => $id }); my $value = $anvil->_get_hash_reference({ key => $id });
print $THIS_FILE." ".__LINE__."; value: [".$value."]\n" if $test;
if (not defined $value) if (not defined $value)
{ {
$string =~ s/#!data!$id!#/!!a[$id]!!/; $string =~ s/#!data!$id!#/!!a[$id]!!/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
else else
{ {
$string =~ s/#!data!$id!#/$value/; $string =~ s/#!data!$id!#/$value/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
} }
else else
{ {
# One dimension # One dimension
print $THIS_FILE." ".__LINE__."; one dimension\n" if $test;
if (not defined $anvil->data->{$id}) if (not defined $anvil->data->{$id})
{ {
$string =~ s/#!data!$id!#/!!b[$id]!!/; $string =~ s/#!data!$id!#/!!b[$id]!!/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
else else
{ {
my $value = $anvil->data->{$id}; my $value = $anvil->data->{$id};
$string =~ s/#!data!$id!#/$value/; $string =~ s/#!data!$id!#/$value/;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
} }
} }
# Die if I've looped too many times. # Die if I've looped too many times.
$loops++; $loops++;
print $THIS_FILE." ".__LINE__."; loops: [".$loops."], limit: [".$limit."]\n" if $test;
if ($loops > $limit) if ($loops > $limit)
{ {
# If we're in a web environment, print the HTML header. # If we're in a web environment, print the HTML header.
print $THIS_FILE." ".__LINE__."; environment: [".$anvil->environment."]\n" if $test;
if ($anvil->environment eq "html") if ($anvil->environment eq "html")
{ {
print "Content-type: text/html; charset=utf-8\n\n"; print "Content-type: text/html; charset=utf-8\n\n";
} }
die "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n"; print "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
} }
$loops++; $loops++;
print $THIS_FILE." ".__LINE__."; loops: [".$loops."], limit: [".$limit."]\n" if $test;
if ($loops > $limit) if ($loops > $limit)
{ {
# If we're in a web environment, print the HTML header. # If we're in a web environment, print the HTML header.
print $THIS_FILE." ".__LINE__."; environment: [".$anvil->environment."]\n" if $test;
if ($anvil->environment eq "html") if ($anvil->environment eq "html")
{ {
print "Content-type: text/html; charset=utf-8\n\n"; print "Content-type: text/html; charset=utf-8\n\n";
} }
die "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n"; print "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
# If there are no replacement keys left, exit the loop. # If there are no replacement keys left, exit the loop.
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
if ($string !~ /#!([^\s]+?)!#/) if ($string !~ /#!([^\s]+?)!#/)
{ {
$loop = 0; $loop = 0;
print $THIS_FILE." ".__LINE__."; loop: [".$loop."]\n" if $test;
} }
} }
@ -1159,27 +1199,34 @@ sub string
while ($loop) while ($loop)
{ {
$string =~ s/!#([^\s]+?)#!/#!$1!#/g; $string =~ s/!#([^\s]+?)#!/#!$1!#/g;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
$loops++; $loops++;
print $THIS_FILE." ".__LINE__."; loops: [".$loops."], limit: [".$limit."]\n" if $test;
if ($loops > $limit) if ($loops > $limit)
{ {
# If we're in a web environment, print the HTML header. # If we're in a web environment, print the HTML header.
print $THIS_FILE." ".__LINE__."; environment: [".$anvil->environment."]\n" if $test;
if ($anvil->environment eq "html") if ($anvil->environment eq "html")
{ {
print "Content-type: text/html; charset=utf-8\n\n"; print "Content-type: text/html; charset=utf-8\n\n";
} }
die "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n"; print "$THIS_FILE ".__LINE__."; Infinite loop detected while processing the string: [".$string."] from the key: [$key] in language: [$language], exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
if ($string !~ /!#[^\s]+?#!/) if ($string !~ /!#[^\s]+?#!/)
{ {
$loop = 0; $loop = 0;
print $THIS_FILE." ".__LINE__."; loop: [".$loop."]\n" if $test;
} }
} }
} }
# In some multi-line strings, the last line will be '\t\t</key>'. We clean this up. # In some multi-line strings, the last line will be '\t\t</key>'. We clean this up.
$string =~ s/\t\t$//; $string =~ s/\t\t$//;
print $THIS_FILE." ".__LINE__."; string: [".$string."]\n" if $test;
#print $THIS_FILE." ".__LINE__."; [ Debug ] - string: [$string]\n"; #print $THIS_FILE." ".__LINE__."; [ Debug ] - string: [$string]\n";
return($string); return($string);

@ -0,0 +1,38 @@
.\" Manpage for the Anvil! server removal tool
.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
.TH anvil-manage-files "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform"
.SH NAME
anvil-manage-files \- This program manages the files sync'ed across machines in the Anvil! cluster
.SH SYNOPSIS
.B anvil-manage-files
\fI\,<command> \/\fR[\fI\,options\/\fR]
.SH DESCRIPTION
This handles synchronizing files, typically ISO used to build new servers and scripts used to manage them, across Anvil! nodes.
.TP
.SH OPTIONS
.TP
\-?, \-h, \fB\-\-help\fR
Show this man page.
.TP
\fB\-\-log-secure\fR
When logging, record sensitive data, like passwords.
.TP
\-v, \-vv, \-vvv
Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
.SS "Commands:"
.TP
\fB\-\-delete\fR
This will delete the \fB\-\-file\fR </path/to/file> from the entire Anvil! cluster.
.TP
This action is permanent!
.TP
\fB\-\-job-uuid\fR <name>
The program is normally run as a job, with data on how to configure the host defined in the job. This switch allows the running of a specific job. If this is not set, the program will search for a job that has not yet been picked up by another process. If found, that job UUID is used automatically.
.IP
.SH AUTHOR
Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
.SH "REPORTING BUGS"
Report bugs to users@clusterlabs.org
", "download", "everywhere", "file", "is-script", "job-uuid", "rename", "to

@ -336,7 +336,11 @@ sub process_resource
"s5:new_scan_drbd_resource_xml" => $new_scan_drbd_resource_xml, "s5:new_scan_drbd_resource_xml" => $new_scan_drbd_resource_xml,
}}); }});
die if not defined $old_scan_drbd_resource_xml; if (not defined $old_scan_drbd_resource_xml)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_drbd_warning_0001"});
$anvil->nice_exit({exit_code => 1});
}
my $update = 0; my $update = 0;
if ($scan_drbd_resource_name ne $old_scan_drbd_resource_name) if ($scan_drbd_resource_name ne $old_scan_drbd_resource_name)
@ -759,7 +763,11 @@ WHERE
scan_drbd_peer_scan_drbd_volume_uuid => $scan_drbd_peer_scan_drbd_volume_uuid, scan_drbd_peer_scan_drbd_volume_uuid => $scan_drbd_peer_scan_drbd_volume_uuid,
}}); }});
die if not $scan_drbd_peer_scan_drbd_volume_uuid; if (not $scan_drbd_peer_scan_drbd_volume_uuid)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_drbd_warning_0002"});
$anvil->nice_exit({exit_code => 1});
}
my $query = " my $query = "
INSERT INTO INSERT INTO

@ -19,6 +19,10 @@ NOTE: All string keys MUST be prefixed with the agent name! ie: 'scan_server_log
<!-- Error entries --> <!-- Error entries -->
<key name="scan_drbd_error_0001">DRBD is not configured on this host, exiting.</key> <key name="scan_drbd_error_0001">DRBD is not configured on this host, exiting.</key>
<!-- Warnings entries -->
<key name="scan_drbd_warning_0001">[ Warning ] - The variable 'old_scan_drbd_resource_xml' is undefined, which should never be the case. This is likely a program error.</key>
<key name="scan_drbd_warning_0002">[ Warning ] - The variable 'scan_drbd_peer_scan_drbd_volume_uuid' is empty, which should never be the case. This is likely a program error.</key>
<!-- Error entries --> <!-- Error entries -->
<key name="scan_drbd_log_0001">Starting The: [#!variable!program!#] DRBD resource agent.</key> <key name="scan_drbd_log_0001">Starting The: [#!variable!program!#] DRBD resource agent.</key>

@ -811,9 +811,16 @@ INSERT INTO
next; next;
} }
### TODO: Make this ia log entry if (not $new_scan_ipmitool_sensor_units)
die "No IPMI units for sensor: [".$scan_ipmitool_sensor_name."]." if not $new_scan_ipmitool_sensor_units; {
die "No IPMI value for sensor: [".$new_scan_ipmitool_value_sensor_value."]." if not $new_scan_ipmitool_value_sensor_value; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_warn_0001", variables => { sensor => $scan_ipmitool_sensor_name }});
next;
}
if (not $new_scan_ipmitool_value_sensor_value)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_ipmitool_warn_0001", variables => { sensor => $new_scan_ipmitool_value_sensor_value }});
next;
}
# If the value is "" and it's a digit-based value, switch it to '0' # If the value is "" and it's a digit-based value, switch it to '0'
if ((($new_scan_ipmitool_sensor_units eq "C") or ($new_scan_ipmitool_sensor_units eq "F") or ($new_scan_ipmitool_sensor_units eq "RPM")) && ($new_scan_ipmitool_value_sensor_value eq "")) if ((($new_scan_ipmitool_sensor_units eq "C") or ($new_scan_ipmitool_sensor_units eq "F") or ($new_scan_ipmitool_sensor_units eq "RPM")) && ($new_scan_ipmitool_value_sensor_value eq ""))

@ -116,5 +116,8 @@ The new sensor: [#!variable!sensor_name!#] has been found on the machine: [#!var
<key name="scan_ipmitool_log_0004">The sensor named: [#!variable!sensor_name!#] appears to have vanished, but this is the first scan that it vanished. This is generally harmless and just a sensor read issue.</key> <key name="scan_ipmitool_log_0004">The sensor named: [#!variable!sensor_name!#] appears to have vanished, but this is the first scan that it vanished. This is generally harmless and just a sensor read issue.</key>
<key name="scan_ipmitool_log_0005">The sensor named: [#!variable!sensor_name!#] has returned.</key> <key name="scan_ipmitool_log_0005">The sensor named: [#!variable!sensor_name!#] has returned.</key>
<!-- Warning entries -->
<key name="scan_ipmitool_warn_0001">[ Warning ] - No IPMI units for sensor: [#!variable!sensor!#].</key>
</language> </language>
</words> </words>

@ -86,6 +86,7 @@ find_changes($anvil);
# Shut down. # Shut down.
$anvil->ScanCore->agent_shutdown({agent => $THIS_FILE}); $anvil->ScanCore->agent_shutdown({agent => $THIS_FILE});
$anvil->nice_exit({exit_code => 0});
############################################################################################################# #############################################################################################################
@ -1087,14 +1088,27 @@ sub collect_pvs_data
{ {
my ($anvil) = @_; my ($anvil) = @_;
my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $anvil->data->{path}{exe}{pvscan}}); my $shell_call = $anvil->data->{path}{exe}{pvscan};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $anvil->data->{path}{exe}{pvs}." --noheadings --units b --reportformat json -o pv_uuid,pv_name,vg_name,pv_attr,pv_size,pv_free"}); $shell_call = $anvil->data->{path}{exe}{pvs}." --noheadings --units b --reportformat json -o pv_uuid,pv_name,vg_name,pv_attr,pv_size,pv_free";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
my $json = JSON->new->allow_nonref; my $json = JSON->new->allow_nonref;
my $pvs_data = $json->decode($output); my $pvs_data = $json->decode($output);
my $default_sector_size = 512;
foreach my $hash_ref (@{$pvs_data->{report}->[0]->{pv}}) foreach my $hash_ref (@{$pvs_data->{report}->[0]->{pv}})
{ {
my $scan_lvm_pv_internal_uuid = $hash_ref->{pv_uuid}; my $scan_lvm_pv_internal_uuid = $hash_ref->{pv_uuid};
@ -1124,15 +1138,19 @@ sub collect_pvs_data
my $sector_size = 0; my $sector_size = 0;
until ($sector_size) until ($sector_size)
{ {
chop $partition;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { partition => $partition }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { partition => $partition }});
if (not $partition) if (not $partition)
{ {
# Weird... Default to 512. # Weird... Default to 512.
$sector_size = 512; $sector_size = $default_sector_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sector_size => $sector_size }}); my $original_partition = $anvil->data->{lvm}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{name};
die; my $sector_path = $directory."/".$original_partition."/queue/hw_sector_size";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_lvm_warning_0001", variables => {
device => $original_partition,
sector_path => $sector_path,
sector_size => $sector_size,
}});
} }
my $sector_size_file = $directory."/".$partition."/queue/hw_sector_size"; my $sector_size_file = $directory."/".$partition."/queue/hw_sector_size";
@ -1146,11 +1164,19 @@ sub collect_pvs_data
if ((not $sector_size) or ($sector_size =~ /\D/)) if ((not $sector_size) or ($sector_size =~ /\D/))
{ {
# Something went wrong, default to 512. # Something went wrong, default to 512.
$sector_size = 512; $sector_size = $default_sector_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sector_size => $sector_size }}); my $original_partition = $anvil->data->{lvm}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{name};
die; my $sector_path = $directory."/".$original_partition."/queue/hw_sector_size";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_lvm_warning_0001", variables => {
device => $original_partition,
sector_path => $sector_path,
sector_size => $sector_size,
}});
} }
} }
# Take a number off and try again.
chop $partition;
} }
$anvil->data->{lvm}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{sector_size} = $sector_size; $anvil->data->{lvm}{scan_lvm_pv_internal_uuid}{$scan_lvm_pv_internal_uuid}{sector_size} = $sector_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
@ -1165,11 +1191,23 @@ sub collect_vgs_data
{ {
my ($anvil) = @_; my ($anvil) = @_;
my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $anvil->data->{path}{exe}{vgscan}}); my $shell_call = $anvil->data->{path}{exe}{vgscan};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $anvil->data->{path}{exe}{vgs}." --noheadings --units b --reportformat json -o vg_uuid,vg_name,vg_attr,vg_extent_size,vg_size,vg_free"}); $shell_call = $anvil->data->{path}{exe}{vgs}." --noheadings --units b --reportformat json -o vg_uuid,vg_name,vg_attr,vg_extent_size,vg_size,vg_free";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
my $json = JSON->new->allow_nonref; my $json = JSON->new->allow_nonref;
my $vgs_data = $json->decode($output); my $vgs_data = $json->decode($output);
@ -1197,11 +1235,23 @@ sub collect_lvs_data
{ {
my ($anvil) = @_; my ($anvil) = @_;
my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $anvil->data->{path}{exe}{lvscan}}); my $shell_call = $anvil->data->{path}{exe}{lvscan};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $anvil->data->{path}{exe}{lvs}." --noheadings --units b --reportformat json -o lv_name,vg_name,lv_attr,lv_size,lv_uuid,lv_path,devices"}); my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
$shell_call = $anvil->data->{path}{exe}{lvs}." --noheadings --units b --reportformat json -o lv_name,vg_name,lv_attr,lv_size,lv_uuid,lv_path,devices";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
my $json = JSON->new->allow_nonref; my $json = JSON->new->allow_nonref;
my $lvs_data = $json->decode($output); my $lvs_data = $json->decode($output);

@ -224,6 +224,9 @@ The attribute bits are:
<!-- Log entries --> <!-- Log entries -->
<key name="scan_lvm_log_0001">Starting: [#!variable!program!#].</key> <key name="scan_lvm_log_0001">Starting: [#!variable!program!#].</key>
<!-- Warning entries -->
<key name="scan_lvm_warning_0001">[ Warning ] - Failed to find the sector size for: [#!variable!device!#] This should be in the path: [#!variable!sector_path!#]. Assuming the sector size of: [#!variable!sector_size!#], but this could be incorrect.</key>
<!-- Message entries (usually meant to be alerts) --> <!-- Message entries (usually meant to be alerts) -->
<key name="scan_lvm_message_0001"></key> <key name="scan_lvm_message_0001"></key>

@ -931,22 +931,7 @@ sub collect_data
closedir(DIRECTORY); closedir(DIRECTORY);
# Find what interfaces are connected to which bridges # Find what interfaces are connected to which bridges
$anvil->Network->bridge_info({debug => 3}); $anvil->Network->bridge_info({debug => 2});
delete $anvil->data->{interface_to_bridge} if exists $anvil->data->{interface_to_bridge};
foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{bridge}{$local_host}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridge_name => $bridge_name }});
foreach my $interface_name (sort {$a cmp $b} @{$anvil->data->{bridge}{$local_host}{$bridge_name}{interfaces}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { interface_name => $interface_name }});
$anvil->data->{interface_to_bridge}{$interface_name} = $bridge_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"interface_to_bridge::${interface_name}" => $anvil->data->{interface_to_bridge}{$interface_name},
}});
}
}
foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}}) foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
{ {
my $ip_address = $anvil->data->{network}{$local_host}{interface}{$interface}{ip}; my $ip_address = $anvil->data->{network}{$local_host}{interface}{$interface}{ip};
@ -4060,7 +4045,11 @@ sub check_bridges
foreach my $bridge_uuid (keys %{$anvil->data->{old}{bridges}{bridge_uuid}}) foreach my $bridge_uuid (keys %{$anvil->data->{old}{bridges}{bridge_uuid}})
{ {
# Skip if already deleted. # Skip if already deleted.
die if not $bridge_uuid; if (not $bridge_uuid)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_network_warning_0001"});
$anvil->nice_exit({exit_code => 1});
}
my $bridge_name = $anvil->data->{old}{bridges}{bridge_uuid}{$bridge_uuid}{bridge_name}; my $bridge_name = $anvil->data->{old}{bridges}{bridge_uuid}{$bridge_uuid}{bridge_name};
my $bridge_id = $anvil->data->{old}{bridges}{bridge_uuid}{$bridge_uuid}{bridge_id}; my $bridge_id = $anvil->data->{old}{bridges}{bridge_uuid}{$bridge_uuid}{bridge_id};

@ -219,5 +219,7 @@ This mode is NOT supported by the Anvil! Intelligent Availability™ platform!
<key name="scan_network_log_0003">The old bond: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key> <key name="scan_network_log_0003">The old bond: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key>
<key name="scan_network_log_0004">The old bridge: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key> <key name="scan_network_log_0004">The old bridge: [#!variable!name!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key>
<key name="scan_network_log_0005">The old IP address: [#!variable!ip!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key> <key name="scan_network_log_0005">The old IP address: [#!variable!ip!#] was marked as deleted more than: [#!variable!age!#] hours ago. Purging it from the database.</key>
<key name="scan_network_warning_0001">There was an empty bridge_uuid in the hash 'old::bridges::bridge_uuid' hash. This is a program error.</key>
</language> </language>
</words> </words>

@ -1664,13 +1664,13 @@ The fingerprint of: [#!variable!machine!#] has changed! Updating it's entry in k
<key name="log_0279">The md5sum of file: [#!variable!file!#] failed to match. Discarding the downloaded file.</key> <key name="log_0279">The md5sum of file: [#!variable!file!#] failed to match. Discarding the downloaded file.</key>
<key name="log_0280">Failed to download: [#!variable!file!#] from: [#!variable!host_name!# (#!variable!ip!#). Will look on other hosts (if any left).</key> <key name="log_0280">Failed to download: [#!variable!file!#] from: [#!variable!host_name!# (#!variable!ip!#). Will look on other hosts (if any left).</key>
<key name="log_0281">The file: [#!variable!file!#] on: [#!variable!host_name!# (#!variable!ip!#]) doesn't match the file we're looking for. <key name="log_0281">The file: [#!variable!file!#] on: [#!variable!host_name!# (#!variable!ip!#]) doesn't match the file we're looking for.
- Wanted; md5sum: [#!variable!file_md5sum!#], size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)] - Wanted; size: [#!variable!say_file_size!# (#!variable!file_size!# bytes)]
- Found; md5sum: [#!variable!remote_md5sum!#], size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)] - Found; size: [#!variable!say_remote_size!# (#!variable!remote_size!# bytes)]
We will keep looking.</key> We will keep looking.</key>
<key name="log_0282">Already searched: [#!variable!host_name!# using another IP address, skipping this IP: [#!variable!ip!#].</key> <key name="log_0282">Already searched: [#!variable!host_name!# using another IP address, skipping this IP: [#!variable!ip!#].</key>
<key name="log_0283">Done.</key> <key name="log_0283">Done.</key>
<key name="log_0284">[ Error ] - Failed to remove the file: [#!variable!file!#]! Please check the permissions or for SELinux denials.</key> <key name="log_0284">[ Error ] - Failed to remove the file: [#!variable!file!#]! Please check the permissions or for SELinux denials.</key>
<key name="log_0285">As requested by another machine, we will now delete the file: [#!variable!file!#].</key> <key name="log_0285">The file: [#!variable!file!#] is marked as not sync'ed to this Anvil!, removing it now.</key>
<key name="log_0286">[ Error ] - The URL: [#!variable!url!#] to download appears to be invalid.</key> <key name="log_0286">[ Error ] - The URL: [#!variable!url!#] to download appears to be invalid.</key>
<key name="log_0287">[ Error ] - The requested URL: [#!variable!url!#] was not found on the remote server.</key> <key name="log_0287">[ Error ] - The requested URL: [#!variable!url!#] was not found on the remote server.</key>
<key name="log_0288">[ Error ] - The requested URL: [#!variable!url!#] does not resolve to a known domain.</key> <key name="log_0288">[ Error ] - The requested URL: [#!variable!url!#] does not resolve to a known domain.</key>
@ -2151,6 +2151,8 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
<key name="log_0721">The server: [#!variable!server!#] is ready to boot.</key> <key name="log_0721">The server: [#!variable!server!#] is ready to boot.</key>
<key name="log_0722">The server: [#!variable!server!#] was found to be running already, but it wasn't marked as booted. Marking it as if it just booted to handle any dependent servers.</key> <key name="log_0722">The server: [#!variable!server!#] was found to be running already, but it wasn't marked as booted. Marking it as if it just booted to handle any dependent servers.</key>
<key name="log_0723">The server: [#!variable!server!#] is configured to stay off, ignoring it.</key> <key name="log_0723">The server: [#!variable!server!#] is configured to stay off, ignoring it.</key>
<key name="log_0724">The file: [#!variable!file!#] needs to be added to the database, but since the last scan it's size grew from: [#!variable!old_size_bytes!# (#!variables!old_size_hr!#)] to: [#!variable!new_size_bytes!# (#!variables!new_size_hr!#)]. A difference of: [#!variable!difference_bytes!# (#!variables!difference_hr!#)]. It might still be being uploaded, so we'll keep checking periodocally until the size stops changing.</key>
<key name="log_0725">Found the missing file: [#!variable!file!#] in the directory: [#!variable!directory!#]. Updating the database now.</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. --> <!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key> <key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>
@ -3248,7 +3250,7 @@ We will sleep a bit and try again.
<key name="warning_0137">[ Warning ] - Timed out waiting for the connections to the peers, and the local resource(s) is not in 'UpToDate' state. Booting the server will likely fail.</key> <key name="warning_0137">[ Warning ] - Timed out waiting for the connections to the peers, and the local resource(s) is not in 'UpToDate' state. Booting the server will likely fail.</key>
<key name="warning_0138">[ Warning ] - Timed out waiting for the connections to the peers.</key> <key name="warning_0138">[ Warning ] - Timed out waiting for the connections to the peers.</key>
<key name="warning_0139">[ Warning ] - We're using: [#!variable!ram_used!#] (#!variable!ram_used_bytes!# Bytes). but there is a job: [#!variable!job_command!#] is runnng, which might be why the RAM is high. NOT exiting while this program is running.</key> <key name="warning_0139">[ Warning ] - We're using: [#!variable!ram_used!#] (#!variable!ram_used_bytes!# Bytes). but there is a job: [#!variable!job_command!#] is runnng, which might be why the RAM is high. NOT exiting while this program is running.</key>
<key name="warning_0140">[ Warning ] - A no-longer active PID: [#!variable!pid!#] had marked the database: [#!variable!db!#] as "in_use", but the PID is gone now. Reaping the flag.</key> <key name="warning_0140">[ Warning ] - A no-longer active PID: [#!variable!pid!#] (used by: [#!variable!caller!#] had marked the database: [#!variable!db!#] as "in_use", but the PID is gone now. Reaping the flag.</key>
<key name="warning_0141">[ Warning ] - We waited for: [#!variable!wait_time!#] seconds for all users of the local database to exit. Giving up waiting and taking the database down now.</key> <key name="warning_0141">[ Warning ] - We waited for: [#!variable!wait_time!#] seconds for all users of the local database to exit. Giving up waiting and taking the database down now.</key>
<key name="warning_0142">[ Warning ] - The command: [#!variable!command!#] is still using our database.</key> <key name="warning_0142">[ Warning ] - The command: [#!variable!command!#] is still using our database.</key>
<key name="warning_0143">[ Warning ] - While evaluating database shutdown, the host UUID: [#!variable!host_uuid!#] was not yet found in the database on host: [#!variable!db_uuid!#]. DB shutdown will not happen until all hosts are in all DBs.</key> <key name="warning_0143">[ Warning ] - While evaluating database shutdown, the host UUID: [#!variable!host_uuid!#] was not yet found in the database on host: [#!variable!db_uuid!#]. DB shutdown will not happen until all hosts are in all DBs.</key>

@ -582,7 +582,7 @@ sub handle_periodic_tasks
my $problem = $anvil->Email->check_config({debug => 3}); my $problem = $anvil->Email->check_config({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
# Check if any files have been uploaded to /mnt/shared/incoming on striker # Check if anything is needed to be done in /mnt/shared.
check_incoming($anvil); check_incoming($anvil);
# Check for stale db_in_use states. # Check for stale db_in_use states.
@ -799,18 +799,30 @@ AND
's3:state_note' => $state_note, 's3:state_note' => $state_note,
}}); }});
my ($db_uuid, $state_pid) = ($state_name =~ /db_in_use::(.*?)::(\d+)$/); my $caller = "";
my ($db_uuid, $state_pid) = ($state_name =~ /db_in_use::(.*?)::(.*)$/);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:db_uuid' => $anvil->Get->host_name_from_uuid({host_uuid => $db_uuid})." (".$db_uuid.")", 's1:db_uuid' => $anvil->Get->host_name_from_uuid({host_uuid => $db_uuid})." (".$db_uuid.")",
's4:state_pid' => $state_pid, 's2:state_pid' => $state_pid,
}}); }});
if ($state_pid =~ /(\d+)::(.*)$/)
{
$state_pid = $1;
$caller = $2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:state_pid' => $state_pid,
's2:caller' => $caller,
}});
}
if (not exists $anvil->data->{pids}{$state_pid}) if (not exists $anvil->data->{pids}{$state_pid})
{ {
# Reap the 'db_is_use'. # Reap the 'db_is_use'.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { state_name => $state_name }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0140", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0140", variables => {
db => $anvil->Get->host_name_from_uuid({host_uuid => $db_uuid})." (".$db_uuid.")", db => $anvil->Get->host_name_from_uuid({host_uuid => $db_uuid})." (".$db_uuid.")",
pid => $state_pid, pid => $state_pid,
'caller' => $caller,
}}); }});
my $query = "DELETE FROM states WHERE state_uuid = ".$anvil->Database->quote($state_uuid).";"; my $query = "DELETE FROM states WHERE state_uuid = ".$anvil->Database->quote($state_uuid).";";
@ -823,73 +835,23 @@ AND
return(0); return(0);
} }
# On dashboards, this checks to see if any files are in /mnt/shared/incoming and, if so, that they've been processed. # This checks to see if any files in /mnt/shared need to be dealt with, like incorporating files in
# /mnt/shared/incoming, etc.
sub check_incoming sub check_incoming
{ {
my ($anvil) = @_; my ($anvil) = @_;
my $system_type = $anvil->Get->host_type(); my $shell_call = $anvil->data->{path}{exe}{'anvil-manage-files'}." --check";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { system_type => $system_type }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
if ($system_type eq "striker") my ($output, $return_code) = $anvil->System->call({
{ shell_call => $shell_call,
# Look for files in /mnt/shared/incoming that are not yet in the database. source => $THIS_FILE,
my $directory = $anvil->data->{path}{directories}{shared}{incoming};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});
local(*DIRECTORY);
opendir(DIRECTORY, $directory);
while(my $file = readdir(DIRECTORY))
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }});
next if $file eq ".";
next if $file eq "..";
next if $file =~ /^\./; # This is files being rsync'ed still
my $full_path = $directory."/".$file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }});
# Skip anything that is not a file.
next if not -f $full_path;
# Is this file already in the DB?
my $query = "SELECT file_uuid FROM files WHERE file_name = ".$anvil->Database->quote($file).";";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
if (not $count)
{
# Add it to the database.
my $size = (stat($full_path))[7];
my $say_size_human = $anvil->Convert->bytes_to_human_readable({'bytes' => $size});
my $say_size_comma = $anvil->Convert->add_commas({number => $size});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
size => $size,
say_size_human => $say_size_human,
say_size_comma => $say_size_comma,
}});
# Register a job to call anvil-sync-shared
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE,
line => __LINE__, line => __LINE__,
job_command => $anvil->data->{path}{exe}{'anvil-sync-shared'},
job_data => "file=".$full_path,
job_name => "storage::move_incoming",
job_title => "job_0132",
job_description => "job_0133",
job_progress => 0,
job_host_uuid => $anvil->data->{sys}{host_uuid},
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
} output => $output,
return_code => $return_code,
} }});
closedir(DIRECTORY);
}
return(0); return(0);
} }

@ -13,10 +13,9 @@
# - If not found anywhere, remove it from 'file_locations' and send an alert. # - If not found anywhere, remove it from 'file_locations' and send an alert.
# - If found, check the size. If it differs, recalculate the md5sum. # - If found, check the size. If it differs, recalculate the md5sum.
# - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'. # - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If # - 4. If called with '--delete', remove from 'file_locations' and all copies on all systems. This is done by
# also used with '--everywhere', then all copies on all systems we know about will be deleted. This is # registering a job against all known hosts. As such, if this is called and the target file doesn't exist,
# done by registering a job against all known hosts. As such, if this is called and the target file # it just clears the job and then exits.
# doesn't exist, it just clears the job and then exits.
# - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit. # - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit.
# #
# Exit codes; # Exit codes;
@ -32,7 +31,7 @@
# - # -
# #
# NOTE: # NOTE:
# - # - remove unsyncs, add syncs.
# #
use strict; use strict;
@ -52,25 +51,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new(); my $anvil = Anvil::Tools->new();
$anvil->data->{switches}{'delete'} = ""; $anvil->Get->switches({list => ["add", "check", "delete", "download", "file", "is-script", "job-uuid", "rename", "remove", "to"], man => $THIS_FILE});
$anvil->data->{switches}{download} = ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->data->{switches}{everywhere} = "";
$anvil->data->{switches}{file} = "";
$anvil->data->{switches}{'is-script'} = "";
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'rename'} = "";
$anvil->data->{switches}{to} = "";
$anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"switches::delete" => $anvil->data->{switches}{'delete'},
"switches::download" => $anvil->data->{switches}{download},
"switches::everywhere" => $anvil->data->{switches}{everywhere},
"switches::file" => $anvil->data->{switches}{file},
"switches::is-script" => $anvil->data->{switches}{'is-script'},
"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'},
"switches::rename" => $anvil->data->{switches}{'rename'},
"switches::to" => $anvil->data->{switches}{to},
}});
# Connect or die # Connect or die
$anvil->Database->connect; $anvil->Database->connect;
@ -124,7 +106,7 @@ elsif ($anvil->data->{switches}{'is-script'})
{ {
handle_script($anvil); handle_script($anvil);
} }
else elsif ($anvil->data->{switches}{check})
{ {
# Check for files scheduled for deletion. # Check for files scheduled for deletion.
check_for_deletes($anvil); check_for_deletes($anvil);
@ -135,6 +117,11 @@ else
# Check for files we should have but don't yet have. # Check for files we should have but don't yet have.
find_missing_files($anvil); find_missing_files($anvil);
} }
else
{
# Show the list of all files we know about.
show_files($anvil);
}
# We're done # We're done
$anvil->nice_exit({exit_code => 0}); $anvil->nice_exit({exit_code => 0});
@ -144,6 +131,16 @@ $anvil->nice_exit({exit_code => 0});
# Private functions. # # Private functions. #
############################################################################################################# #############################################################################################################
# This lists all files.
sub show_files
{
my ($anvil) = @_;
return(0);
}
# This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't # This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't
# exist on this machine. For those entries, we will search for the file on other machines. The one exception # exist on this machine. For those entries, we will search for the file on other machines. The one exception
# is server definition files. For those, we write the file out from the server's 'definition' table entry. # is server definition files. For those, we write the file out from the server's 'definition' table entry.
@ -169,33 +166,28 @@ sub find_missing_files
my ($anvil) = @_; my ($anvil) = @_;
# What am I? This will impact how missing files are found. # What am I? This will impact how missing files are found.
$anvil->Database->get_anvils();
my $query = " my $query = "
SELECT SELECT
a.file_uuid, file_uuid,
a.file_directory, file_directory,
a.file_name, file_name,
a.file_size, file_size,
a.file_md5sum file_md5sum
FROM FROM
files a, files
hosts b,
file_locations c
WHERE WHERE
b.host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." file_type != 'DELETED'
AND
a.file_uuid = c.file_location_file_uuid
AND
b.host_uuid = c.file_location_host_uuid
ORDER BY ORDER BY
a.file_directory ASC, file_directory ASC,
a.file_name ASC, file_name ASC
b.host_name ASC
;"; ;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results, results => $results,
count => $count, count => $count,
}}); }});
@ -206,262 +198,196 @@ ORDER BY
my $file_name = $row->[2]; my $file_name = $row->[2];
my $file_size = $row->[3]; my $file_size = $row->[3];
my $file_md5sum = $row->[4]; my $file_md5sum = $row->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { my $full_path = $file_directory."/".$file_name;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:file_uuid' => $file_uuid, 's1:file_uuid' => $file_uuid,
's2:file_directory' => $file_directory, 's2:file_directory' => $file_directory,
's3:file_name' => $file_name, 's3:file_name' => $file_name,
's4:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
's5:file_md5sum' => $file_md5sum,
's6:full_path' => $full_path,
}}); }});
my $test_file = $file_directory."/".$file_name; # If we're a striker, we want all files.
$test_file =~ s/\/\//\//g; my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_file => $test_file }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type eq "striker")
if (not -e $test_file) {
if (not -e $full_path)
{ {
# Missing file! # Missing file!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $test_file }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $full_path }});
# Find what target, if any, we'll the file from. # Find what target, if any, we'll the file from.
my ($found) = find_file($anvil, $file_uuid, $test_file, $file_size, $file_md5sum); my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
} }
# If we're a Striker, see if any Anvil! systems exist that don't know about this file.
$anvil->Database->check_file_locations({debug => 2});
} }
else
{
# Check to see if we're supposed to have this file.
$anvil->Database->get_file_locations();
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
### TODO: Left off here. # Nothing to do if we're not in an Anvil! yet.
next if not $anvil_uuid;
return(0); # Do we have a file_location_uuid? If not, there will be soon but nothing to do until
} # then.
my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
next if not $file_location_uuid;
### TODO: Add sorting by preferred target # Should we have it?
# This looks for a file on another system. The exact order of search depends on what kind of machine we are. my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
sub find_file $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_active => $file_location_active }});
if ($file_location_active)
{ {
my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_; # Make sure it exists, and add it if not.
if (not -e $full_path)
{
# Find and copy it.
my ($found) = find_file($anvil, $file_uuid, $full_path, $file_size, $file_md5sum);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
}
}
else
{
# If it exists, remove it.
if (-e $full_path)
{
# Delete it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }});
my $found = 0; unlink $full_path;
# What are my IPs? # Sleep and verify
my $local_host = $anvil->Get->short_host_name(); sleep 1;
$anvil->Network->get_ips(); if (-e $full_path)
foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}})
{ {
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{ip}; # Failed to delete...
next if not $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask}; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }});
my $ip = $anvil->data->{network}{$local_host}{interface}{$interface}{ip}; return("");
my $subnet_mask = $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask}; }
my $network = $anvil->Network->get_network({ip => $ip, subnet_mask => $subnet_mask}); else
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:interface' => $interface,
's2:ip' => $ip,
's3:subnet_mask' => $subnet_mask,
's4:network' => $network,
}});
my $type = "other";
my $sort = $interface;
if ($interface =~ /^((?:bc|s|if)n)(\d+)/)
{ {
$type = $1; # Deleted successfully.
$sort = $2; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { }
's1:type' => $type, }
's2:sort' => $sort, }
}});
} }
$anvil->data->{local_ip}{by_network}{$network}{type} = $type;
$anvil->data->{local_ip}{by_network}{$network}{'sort'} = $sort;
$anvil->data->{local_ip}{by_network}{$network}{interface} = $interface;
$anvil->data->{local_ip}{by_network}{$network}{ip} = $ip;
$anvil->data->{local_ip}{by_network}{$network}{subnet_mask} = $subnet_mask;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:local_ip::by_network::${network}::type" => $anvil->data->{local_ip}{by_network}{$network}{type},
"s1:local_ip::by_network::${network}::sort" => $anvil->data->{local_ip}{by_network}{$network}{'sort'},
"s2:local_ip::by_network::${network}::interface" => $anvil->data->{local_ip}{by_network}{$network}{interface},
"s3:local_ip::by_network::${network}::ip" => $anvil->data->{local_ip}{by_network}{$network}{ip},
"s4:local_ip::by_network::${network}::subnet_mask" => $anvil->data->{local_ip}{by_network}{$network}{subnet_mask},
}});
} }
# Create a hash we can sort through that are on the same subnet_mask as us. return(0);
my $query = " }
SELECT
a.host_uuid, # This looks for a file on another system. The exact order of search depends on what kind of machine we are.
a.host_name, sub find_file
a.host_type,
b.file_directory,
b.file_name,
b.file_size,
b.file_md5sum
FROM
hosts a,
files b,
file_locations c
WHERE
a.host_uuid = c.file_location_host_uuid
AND
b.file_uuid = c.file_location_file_uuid
AND
b.file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
a.host_uuid != ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
ORDER BY
file_mtime DESC;
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{ {
my $host_uuid = $row->[0]; my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_;
my $host_name = $row->[1]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
my $host_type = $row->[2]; 's1:file_uuid' => $file_uuid,
my $file_directory = $row->[3]; 's2:full_path' => $full_path,
my $file_name = $row->[4]; 's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
my $file_size = $row->[5]; 's4:file_md5sum' => $file_md5sum,
my $file_md5sum = $row->[6];
my $test_file = $file_directory."/".$file_name;
$test_file =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_uuid => $host_uuid,
host_name => $host_name,
host_type => $host_type,
file_directory => $file_directory,
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
test_file => $test_file,
}}); }});
# What IP addresses are on this machine? # Before we do anything, see if the file is shown as being in a directory other than 'files' in the
my $query = " # database, but actually in files on disk. If so, update the database.
SELECT my $host_type = $anvil->Get->host_type();
ip_address_address, my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
ip_address_subnet_mask my $file_name = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
FROM my $files_full_path = $anvil->data->{path}{directories}{shared}{files}."/".$file_name;
ip_addresses $files_full_path =~ s/\/\//\//g;
WHERE $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
ip_address_note != 'DELETED' 's1:file_directory' => $file_directory,
AND 's2:file_name' => $file_name,
ip_address_host_uuid = ".$anvil->Database->quote($host_uuid)." 's3:files_full_path' => $files_full_path,
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}}); }});
foreach my $row (@{$results})
if (($full_path ne $files_full_path) && (-e $files_full_path) && ($host_type eq "striker"))
{ {
my $ip_address_address = $row->[0]; ### TODO: Add checks to see if anything other than the directory needs updating.
my $ip_address_subnet_mask = $row->[1]; # It's in 'files' now, update the database.
my $network = $anvil->Network->get_network({ip => $ip_address_address, subnet_mask => $ip_address_subnet_mask}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0725", variables => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file => $full_path,
ip_address_address => $ip_address_address, directory => $anvil->data->{path}{directories}{shared}{files},
ip_address_subnet_mask => $ip_address_subnet_mask,
network => $network,
}}); }});
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{files}{file_uuid}{$file_uuid}{file_name},
file_directory => $anvil->data->{path}{directories}{shared}{files},
file_size => $anvil->data->{files}{file_uuid}{$file_uuid}{file_size},
file_md5sum => $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum},
file_mtime => $anvil->data->{files}{file_uuid}{$file_uuid}{file_mtime},
file_type => $anvil->data->{files}{file_uuid}{$file_uuid}{file_type},
});
# Are we on the same subnet? return(1);
if (exists $anvil->data->{local_ip}{by_network}{$network}) }
# We want to search Striker's first, DR hosts second and nodes third.
$anvil->Database->get_hosts;
my $host_order = [];
foreach my $type ("striker", "dr", "node")
{ {
# We're on the same subnet! $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type }});
my $type = $anvil->data->{local_ip}{by_network}{$network}{type}; foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}})
my $sort = $anvil->data->{local_ip}{by_network}{$network}{'sort'}; {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name};
"s1:type" => $type, my $host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type};
"s2:sort" => $sort, $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:host_name' => $host_name,
's2:host_type' => $host_type,
's3:host_uuid' => $host_uuid,
}}); }});
# Record. next if $host_type ne $type;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip} = $ip_address_address; next if $host_uuid eq $anvil->Get->host_uuid;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name} = $host_name;
$anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid} = $host_uuid; push @{$host_order}, $host_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
"s1:peer_ip::${host_type}::${type}::${sort}::name" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name},
"s2:peer_ip::${host_type}::${type}::${sort}::ip" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip},
"s3:peer_ip::${host_type}::${type}::${sort}::host_uuid" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid},
}});
}
} }
} }
### TODO: Do this according to what we are. # Now search.
# Sort through what we've found. my $found_on_host = 0; # Found the file on a target
my $file_found = 0; my $file_found = 0; # Actually have the file locally.
my $searched_hosts = {}; foreach my $search_host_uuid (@{$host_order})
foreach my $host_type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}})
{ {
last if $file_found; last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }}); my $target_host = $anvil->data->{hosts}{host_uuid}{$search_host_uuid}{short_host_name};
foreach my $type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}})
{
last if $file_found;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }});
foreach my $sort (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}{$type}})
{
last if $file_found;
my $ip = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip};
my $name = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name};
my $host_uuid = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid};
my $remote_user = "admin";
my $password = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:host_type' => $host_type, 's1:target_host' => $target_host,
's2:type' => $type, 's2:search_host_uuid' => $search_host_uuid,
's3:sort' => $sort,
's4:name' => $name,
's5:ip' => $ip,
's6:host_uuid' => $host_uuid,
's7:remote_user' => $remote_user,
's8:password' => $password,
}}); }});
if ((exists $searched_hosts->{$name}) && ($searched_hosts->{$name})) my $target_ip = $anvil->Network->find_target_ip({host_uuid => $search_host_uuid});
{ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_ip => $target_ip }});
# Already searched this host, this is just a different IP. Skip it. next if not $target_ip;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0282", variables => {
host_name => $name,
ip => $ip,
}});
next;
}
$searched_hosts->{$name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "searched_hosts->{$name}" => $searched_hosts->{$name} }});
### NOTE: There's a bug in Net::SSH2 on RHEL8 where passwordless SSH doesn't # See if the file is on the target and, if so, if it matches.
### work. So for now, we'll manually pull in passwords from the ### NOTE: If we want to use md5sum again, use '--with-md5sum'
### anvil.conf using 'hosts::<host_name>::password' or my $shell_call = $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path.$anvil->Log->switches;
### 'hosts::<host_uuid>::password'. This will be removed when the bug $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
### is fixed.
if ((exists $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}) && ($anvil->data->{hosts}{$host_uuid}{password}{$remote_user})) # Test access
{ my $access = $anvil->Remote->test_access({target => $target_ip});
$password = $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }}); next if not $access;
}
elsif ((exists $anvil->data->{hosts}{$name}{password}{$remote_user}) && ($anvil->data->{hosts}{$name}{password}{$remote_user}))
{
$password = $anvil->data->{hosts}{$host_uuid}{password};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }});
}
### NOTE: This might take a while for the call to return if we're md5sum'ing
### a large file like an ISO.
### TODO: Should we ask for the size, then follow up with the md5sum for
### files that are over a certain size? Or will this get called rarely
### enough in practice that it doesn't really matter?
# If the file is found, we'll parse these out.
my $remote_size = 0; my $remote_size = 0;
my $remote_md5sum = "";
my ($output, $error, $return_code) = $anvil->Remote->call({ my ($output, $error, $return_code) = $anvil->Remote->call({
shell_call => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum".$anvil->Log->switches, shell_call => $shell_call,
remote_user => $remote_user, target => $target_ip,
password => $password,
target => $ip,
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error, error => $error,
@ -470,39 +396,36 @@ AND
}}); }});
foreach my $line (split/\n/, $output) foreach my $line (split/\n/, $output)
{ {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /^size: \[(\d+)\]$/) if ($line =~ /^size: \[(\d+)\]$/)
{ {
$remote_size = $1; $remote_size = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_size => $remote_size }}); $found_on_host = 1;
} $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
if ($line =~ /^md5sum: \[(.*)\]$/) remote_size => $remote_size,
{ found_on_host => $found_on_host,
$remote_md5sum = $1; }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_md5sum => $remote_md5sum }});
} }
} }
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:remote_size' => $remote_size, 's1:found_on_host' => $found_on_host,
's2:file_size' => $file_size, 's2:remote_size' => $remote_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}).")",
's3:remote_md5sum' => $remote_md5sum, 's3:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
's4:file_md5sum' => $file_md5sum,
}}); }});
next if not $found_on_host;
### Do I really need to match sizes if the md5sum is the same? if ($remote_size eq $file_size)
if (($remote_size eq $file_size) && ($remote_md5sum eq $file_md5sum))
{ {
# Pull it over! # Pull it over!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => {
file => $full_path, file => $full_path,
host_name => $name, host_name => $target_host,
ip => $ip, ip => $target_ip,
}}); }});
my $failed = $anvil->Storage->rsync({ my $failed = $anvil->Storage->rsync({
debug => 2, debug => 2,
destination => $full_path, destination => $full_path,
password => $password, source => "root\@".$target_ip.":".$full_path,
source => $remote_user."\@".$ip.":".$full_path,
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
@ -538,8 +461,8 @@ AND
# Failed to rsync. # Failed to rsync.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => {
file => $full_path, file => $full_path,
host_name => $name, host_name => $target_host,
ip => $ip, ip => $target_ip,
}}); }});
} }
} }
@ -548,10 +471,8 @@ AND
# Doesn't match what we're looking for. # Doesn't match what we're looking for.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => {
file => $full_path, file => $full_path,
host_name => $name, host_name => $target_host,
ip => $ip, ip => $target_ip,
remote_md5sum => $remote_md5sum,
file_md5sum => $file_md5sum,
file_size => $file_size, file_size => $file_size,
say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}), say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}),
remote_size => $remote_size, remote_size => $remote_size,
@ -559,11 +480,9 @@ AND
}}); }});
} }
} }
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }});
return($found); return($file_found);
} }
@ -577,6 +496,7 @@ sub check_incoming
{ {
delete $anvil->data->{scan}{directories}; delete $anvil->data->{scan}{directories};
} }
# Read any files in '/mnt/shared/incoming'. # Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({ $anvil->Storage->scan_directory({
debug => 3, debug => 3,
@ -584,16 +504,28 @@ sub check_incoming
recursive => 1, recursive => 1,
}); });
my $archives_directory = $anvil->data->{path}{directories}{shared}{archives};
my $definitions_directory = $anvil->data->{path}{directories}{shared}{definitions};
my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming}; my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming};
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"}); my $provision_directory = $anvil->data->{path}{directories}{shared}{provision};
my $temp_directory = $anvil->data->{path}{directories}{shared}{temp};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
archives_directory => $archives_directory,
definitions_directory => $definitions_directory,
incoming_directory => $incoming_directory,
provision_directory => $provision_directory,
temp_directory => $temp_directory,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0264"});
foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}}) foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}})
{ {
# Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc) # Skip this if it's under a directory managed elsewhere, or that we don't care about.
next if $full_path =~ /^\/mnt\/shared\/temp\//; next if $full_path =~ /^$temp_directory/;
next if $full_path =~ /^$provision_directory/;
next if $full_path =~ /^$definitions_directory/;
# Skip if this isn't a file. # Skip if this isn't a file.
my $file_type = $anvil->data->{scan}{directories}{$full_path}{type}; my $file_type = $anvil->data->{scan}{directories}{$full_path}{type};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path, full_path => $full_path,
file_type => $file_type, file_type => $file_type,
@ -607,6 +539,8 @@ sub check_incoming
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0; my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable); my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable);
my $full_path = $file_directory."/".$file_name;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file_name => $file_name, file_name => $file_name,
file_size => $file_size, file_size => $file_size,
@ -614,6 +548,7 @@ sub check_incoming
file_mimetype => $file_mimetype, file_mimetype => $file_mimetype,
file_executable => $file_executable, file_executable => $file_executable,
say_mimetype => $say_mimetype, say_mimetype => $say_mimetype,
full_path => $full_path,
}}); }});
# Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum. # Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
@ -629,8 +564,19 @@ sub check_incoming
my $file_md5sum = $recorded_md5sum; my $file_md5sum = $recorded_md5sum;
if ((not $file_uuid) or ($file_size != $recorded_size)) if ((not $file_uuid) or ($file_size != $recorded_size))
{ {
# Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer # It's possible the file is still uploading, so sleep for 2 seconds and see if the
# version down from elsewhere. # size is still changing.
my $last_size = $file_size;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
last_size => $last_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $last_size}).")",
}});
$anvil->Storage->_wait_if_changing({
file => $full_path,
last_size => $file_size,
});
# Yes, caluclate md5sum, but first, do we have a size mismatch? If so, see if we need
# to pull a newer version down from elsewhere.
if (($file_uuid) && ($file_mtime <= $recorded_mtime)) if (($file_uuid) && ($file_mtime <= $recorded_mtime))
{ {
# We've got an older file, we need to update. # We've got an older file, we need to update.
@ -675,14 +621,6 @@ sub check_incoming
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }});
next if not $file_uuid; next if not $file_uuid;
# Make sure we know about this file on this system
my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({
debug => 3,
file_location_file_uuid => $file_uuid,
file_location_host_uuid => $anvil->data->{sys}{host_uuid},
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_locatiom_uuid => $file_locatiom_uuid }});
# Are we in the incoming directory? If so, move the file. # Are we in the incoming directory? If so, move the file.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
full_path => $full_path, full_path => $full_path,
@ -690,6 +628,9 @@ sub check_incoming
}}); }});
if ($full_path =~ /^$incoming_directory/) if ($full_path =~ /^$incoming_directory/)
{ {
# Ignore files starting with '.', they're often in-progress rsync's.
next if $file_name =~ /^\./;
# If it's a definition file, we'll move it to # If it's a definition file, we'll move it to
# 'path::directories::shared::definitions', otherwise we'll move it to # 'path::directories::shared::definitions', otherwise we'll move it to
# 'path::directories::shared::files'. # 'path::directories::shared::files'.
@ -702,6 +643,13 @@ sub check_incoming
file => $full_path, file => $full_path,
target => $target, target => $target,
}}); }});
# Wait in case it's still being uploaded.
$anvil->Storage->_wait_if_changing({
file => $full_path,
last_size => $file_size,
});
$anvil->Storage->move_file({ $anvil->Storage->move_file({
debug => 3, debug => 3,
source_file => $full_path, source_file => $full_path,
@ -709,12 +657,11 @@ sub check_incoming
}); });
# Update the file_directory. # Update the file_directory.
my $say_directory =~ s/\/$//;
($file_uuid) = $anvil->Database->insert_or_update_files({ ($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 3, debug => 3,
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_name => $file_name, file_name => $file_name,
file_directory => $say_directory, file_directory => $target,
file_size => $file_size, file_size => $file_size,
file_md5sum => $file_md5sum, file_md5sum => $file_md5sum,
file_mtime => $file_mtime, file_mtime => $file_mtime,
@ -975,47 +922,31 @@ sub convert_mimetype
return($say_mimetype); return($say_mimetype);
} }
# This looks for any files with the file_type of 'DELETED' and that has an entry in file_locations for this # This looks for any files with the file_type of 'DELETED'. Any that are found will be deleted.
# machine. Any that are found will have their file deleted and the file_locations entry deleted.
sub check_for_deletes sub check_for_deletes
{ {
my ($anvil) = @_; my ($anvil) = @_;
my $query = " # Get a list of files.
SELECT my $query = "SELECT file_uuid, file_directory || '/' || file_name AS full_path FROM files WHERE file_type = 'DELETED';";
a.file_uuid, $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
b.file_location_uuid,
a.file_directory || '/' || a.file_name AS full_path
FROM
files a,
file_locations b
WHERE
a.file_uuid = b.file_location_file_uuid
AND
a.file_type = 'DELETED'
AND
b.file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results, results => $results,
count => $count, count => $count,
}}); }});
foreach my $row (@{$results}) foreach my $row (@{$results})
{ {
my $file_uuid = $row->[0]; my $file_uuid = $row->[0];
my $file_location_uuid = $row->[1]; my $full_path = $row->[1];
my $full_path = $row->[2]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:file_uuid' => $file_uuid, 's1:file_uuid' => $file_uuid,
's2:file_location_uuid' => $file_location_uuid, 's2:full_path' => $full_path,
's3:full_path' => $full_path,
}}); }});
# Get rid of it (if it actually exists) and then remove it from file_locations. # Get rid of it (if it actually exists).
if (-e $full_path) if (-e $full_path)
{ {
# Delete it. # Delete it.
@ -1038,11 +969,36 @@ AND
} }
} }
# If we're a Striker, check for any file_locations that point to this file and DELETE them.
my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type eq "striker")
{
my $query = "SELECT file_location_uuid FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_location_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
# Delete the entry from file_locations, if needed. # Delete the entry from file_locations, if needed.
my $query = "DELETE FROM history.file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
$query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";"; $query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
} }
}
}
return(0); return(0);
} }
@ -1061,7 +1017,7 @@ sub handle_delete
# Um... # Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
$anvil->Job->update_progress({ $anvil->Job->update_progress({
progress => 0, progress => 100,
message => "error_0052", message => "error_0052",
job_uuid => $anvil->data->{jobs}{'job-uuid'}, job_uuid => $anvil->data->{jobs}{'job-uuid'},
}); });
@ -1072,7 +1028,7 @@ sub handle_delete
# We don't do that here... # We don't do that here...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }});
$anvil->Job->update_progress({ $anvil->Job->update_progress({
progress => 0, progress => 100,
message => "error_0053,!!file!".$full_path."!!", message => "error_0053,!!file!".$full_path."!!",
job_uuid => $anvil->data->{jobs}{'job-uuid'}, job_uuid => $anvil->data->{jobs}{'job-uuid'},
}); });
@ -1126,7 +1082,7 @@ AND
# database. This way, if anything goes wrong below, future runs will try (again) to delete the file # database. This way, if anything goes wrong below, future runs will try (again) to delete the file
# itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt # itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt
# to try again. # to try again.
if (($file_uuid) && ($anvil->data->{switches}{everywhere})) if ($file_uuid)
{ {
# Yup. # Yup.
($file_uuid) = $anvil->Database->insert_or_update_files({ ($file_uuid) = $anvil->Database->insert_or_update_files({
@ -1164,18 +1120,13 @@ AND
} }
} }
# Now, if I have a file_uuid, delete it from file_locations if it exists. # Now, if I have a file_uuid, delete it from file_locations if it exists. It's ok if peers in an
# Anvil! haven't deleted yet, as they'll see the file marked as 'DELETED' and purge their local
# copies.
if ($file_uuid) if ($file_uuid)
{ {
# Delete the entry from file_locations, if needed. # Delete the entry from file_locations, if needed.
my $query = " my $query = "DELETE FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid).";";
DELETE FROM
file_locations
WHERE
file_location_file_uuid = ".$anvil->Database->quote($file_uuid)."
AND
file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
} }

@ -168,6 +168,10 @@ sub run_jobs
} }
else else
{ {
# Make sure we're actually in the cluster now.
if ($anvil->data->{cib}{parsed}{'local'}{ready})
{
# We're ready!
$waiting = 0; $waiting = 0;
$anvil->Job->update_progress({ $anvil->Job->update_progress({
progress => 8, progress => 8,
@ -175,6 +179,16 @@ sub run_jobs
}); });
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0276"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0276"});
} }
else
{
# Cluster is coming up, but it's not up yet.
$anvil->Job->update_progress({
progress => 6,
message => "job_0278",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0278"});
}
}
} }
# Sanity checks passed # Sanity checks passed

@ -77,12 +77,6 @@ if ($anvil->data->{switches}{'job-uuid'})
process_file_mode($anvil); process_file_mode($anvil);
} }
} }
else
{
# This checks to see if there are any files that need to be pulled, renamed, removed, or mode
# updated.
#run_checks($anvil);
}
$anvil->nice_exit({exit_code => 0}); $anvil->nice_exit({exit_code => 0});
@ -91,72 +85,6 @@ $anvil->nice_exit({exit_code => 0});
# Functions # # Functions #
############################################################################################################# #############################################################################################################
### TODO: Left off here. Run periodic checks for files that need to be pulled/removed, mode changed,
### etc. For 3.0, we'll ignore files people add manually though later we'll probably want to
### auto-sync them.
### NOTE: When finding new files, check the size, sleep for 30 seconds, and check again. If a file's
### size changed, skip it, it's likely still being updated.
### NOTE: This isn't finished, and likely won't be for a while.
sub run_checks
{
my ($anvil) = @_;
# First, get a list of files we're expected to have.
$anvil->Database->get_files({debug => 3, include_deleted => 1});
$anvil->Database->get_file_locations({debug => 3});
if ($anvil->Get->host_type eq "striker")
{
# Look for files to add
}
else
{
# Are we an Anvil! member?
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
if (not $anvil_uuid)
{
# Nothing to do.
$anvil->nice_exit({exit_code => 0});
}
# What files should we have?
foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
{
# TODO: Once per day, calculate the md5sums and compare against the DB.
my $file_uuid = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
my $file_directory = $anvil->data->{files}{file_name}{$file_name}{file_directory};
my $file_size = $anvil->data->{files}{file_name}{$file_name}{file_size};
my $file_type = $anvil->data->{files}{file_name}{$file_name}{file_type};
my $file_path = $file_directory."/".$file_name;
my $file_location_uuid = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
file_type => $file_type,
file_path => $file_path,
file_location_uuid => $file_location_uuid,
file_location_active => $file_location_active,
}});
if (-e $file_path)
{
# File exists, should it?
}
else
{
# File doesn't exist, should it?
}
}
}
return(0);
}
sub process_file_mode sub process_file_mode
{ {
my ($anvil) = @_; my ($anvil) = @_;

@ -553,20 +553,6 @@ sub update_network
# Find what interfaces are connected to which bridges # Find what interfaces are connected to which bridges
$anvil->Network->bridge_info({debug => 2}); $anvil->Network->bridge_info({debug => 2});
delete $anvil->data->{interface_to_bridge} if exists $anvil->data->{interface_to_bridge};
foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{bridge}{$local_host}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { bridge_name => $bridge_name }});
foreach my $interface_name (sort {$a cmp $b} @{$anvil->data->{bridge}{$local_host}{$bridge_name}{interfaces}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { interface_name => $interface_name }});
$anvil->data->{interface_to_bridge}{$interface_name} = $bridge_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"interface_to_bridge::${interface_name}" => $anvil->data->{interface_to_bridge}{$interface_name},
}});
}
}
# We need to record bridges first so we know their UUIDs when looking at bonds and interfaces that # We need to record bridges first so we know their UUIDs when looking at bonds and interfaces that
# might be connected to them. Then we need to look at bonds so that their UUIDs are available when # might be connected to them. Then we need to look at bonds so that their UUIDs are available when

@ -74,3 +74,4 @@ else
$anvil->nice_exit({ exit_code => 1 }); $anvil->nice_exit({ exit_code => 1 });
} }
$anvil->nice_exit({ exit_code => 0 });

Loading…
Cancel
Save