#!/usr/bin/perl # # This handles moving around and managing files on Anvil! nodes, DR hosts and Striker dashboards. # # When this is called (periodically by the daemon of after an upload / ISO generation); # - 1. Check 'incoming/' for files. For any found, generate an md5sum and see if the file name and sum match # anything in the database from any host; # - If so, update/add an entry in 'file_locations' # - If not, create a new entry in 'files' and then add the first entry in 'file_locations' # - 2. Check 'file_locations' for any files on this system, and verify they exist still. # - If not, check the other files for one with a matching md5sum. If found, handle as a rename. # - If not found at all, search for the file according to the search rules and copy to here if found. # - If not found anywhere, remove it from 'file_locations' and send an alert. # - If found, check the size. If it differs, recalculate the md5sum. # - 3. If called with '--rename --file --to ', rename the file and update 'files'. # - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If # also used with '--everywhere', then all copies on all systems we know about will be deleted. This is # done by registering a job against all known hosts. As such, if this is called and the target file # doesn't exist, it just clears the job and then exits. # - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit. # # Exit codes; # 0 = Normal exit or md5sum of this program changed and it exited to reload. # 1 = No databases available. # 2 = Another process that is still running has picked up this job. # 3 = No filename specified when needed by another switch. # 4 = Request to change the file name but the new name wasn't given. # 5 = The file specified was not found. # 6 = The file to delete is not under '/mnt/shared/'. # # TODO: # - # # NOTE: # - # use strict; use warnings; use Anvil::Tools; use Data::Dumper; # Disable buffering $| = 1; my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; if (($running_directory =~ /^\./) && ($ENV{PWD})) { $running_directory =~ s/^\./$ENV{PWD}/; } my $anvil = Anvil::Tools->new(); $anvil->data->{switches}{'delete'} = ""; $anvil->data->{switches}{download} = ""; $anvil->data->{switches}{everywhere} = ""; $anvil->data->{switches}{file} = ""; $anvil->data->{switches}{'is-script'} = ""; $anvil->data->{switches}{'job-uuid'} = ""; $anvil->data->{switches}{'rename'} = ""; $anvil->data->{switches}{to} = ""; $anvil->Get->switches; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "switches::delete" => $anvil->data->{switches}{'delete'}, "switches::download" => $anvil->data->{switches}{download}, "switches::everywhere" => $anvil->data->{switches}{everywhere}, "switches::file" => $anvil->data->{switches}{file}, "switches::is-script" => $anvil->data->{switches}{'is-script'}, "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'}, "switches::rename" => $anvil->data->{switches}{'rename'}, "switches::to" => $anvil->data->{switches}{to}, }}); # Connect or die $anvil->Database->connect; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{database}{connections}) { # No databases, exit. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"}); $anvil->nice_exit({exit_code => 1}); } # If we have a job_uuid, pick it up. if ($anvil->data->{switches}{'job-uuid'}) { my $return = $anvil->Job->get_job_details({check => 1, job_uuid => $anvil->data->{switches}{'job-uuid'}}); if ($return == 1) { # It's not a UUID. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }}); $anvil->nice_exit({exit_code => 2}); } if ($return == 2) { # This job is being handled by another process that is still alive. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }}); $anvil->nice_exit({exit_code => 3}); } # If there's a progress, clear it. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'job::job_progress' => $anvil->data->{job}{job_progress} }}); if ($anvil->data->{job}{job_progress}) { $anvil->Job->update_progress({ progress => 1, message => "clear", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); } } # What are we doing? if ($anvil->data->{switches}{'rename'}) { handle_rename($anvil); } elsif ($anvil->data->{switches}{'delete'}) { handle_delete($anvil); } elsif ($anvil->data->{switches}{'is-script'}) { handle_script($anvil); } else { # Check for files scheduled for deletion. check_for_deletes($anvil); # Check for new files check_incoming($anvil); # Check for files we should have but don't yet have. find_missing_files($anvil); } # We're done $anvil->nice_exit({exit_code => 0}); ############################################################################################################# # Private functions. # ############################################################################################################# # This looks to see if there are any entries in 'file_locations' for us that, the pointed to file, doesn't # exist on this machine. For those entries, we will search for the file on other machines. The one exception # is server definition files. For those, we write the file out from the server's 'definition' table entry. # # When a missing entry is found, and an entry doesn't exist, the file will be found (if possible) and copied # to this host. Only machines on the same subnet are searched. The search pattern is; # # Nodes; 1. Check for the file on the peer. # 2. Check for the file on Strikers, in alphabetical order. # 3. Check for the file on DR host, if available. # 4. Check other nodes, in alphabetical order. # 5. Check other DR hosts, in alphabetical order. # Striker; 1. Check for the file on other Strikers, in alphabetical order. # 2. Check for the file on DR hosts, if available # 3. Check for the file on Anvil! nodes. # DR Host; 1. Check for the file on Strikers, in alphabetical order. # 2. Check for the file on Anvil! nodes. # * If a file can't be found, it will try again every so often until it is found. # * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and the # md5sum matches. At this point, it is moved into the proper directory. sub find_missing_files { my ($anvil) = @_; # What am I? This will impact how missing files are found. my $query = " SELECT a.file_uuid, a.file_directory, a.file_name, a.file_size, a.file_md5sum FROM files a, hosts b, file_locations c WHERE b.host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." AND a.file_uuid = c.file_location_file_uuid AND b.host_uuid = c.file_location_host_uuid ORDER BY a.file_directory ASC, a.file_name ASC, b.host_name ASC ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { results => $results, count => $count, }}); foreach my $row (@{$results}) { my $file_uuid = $row->[0]; my $file_directory = $row->[1]; my $file_name = $row->[2]; my $file_size = $row->[3]; my $file_md5sum = $row->[4]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 's1:file_uuid' => $file_uuid, 's2:file_directory' => $file_directory, 's3:file_name' => $file_name, }}); my $test_file = $file_directory."/".$file_name; $test_file =~ s/\/\//\//g; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_file => $test_file }}); if (not -e $test_file) { # Missing file! $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0269", variables => { file => $test_file }}); # Find what target, if any, we'll the file from. my ($found) = find_file($anvil, $file_uuid, $test_file, $file_size, $file_md5sum); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); } } ### TODO: Left off here. return(0); } ### TODO: Add sorting by preferred target # This looks for a file on another system. The exact order of search depends on what kind of machine we are. sub find_file { my ($anvil, $file_uuid, $full_path, $file_size, $file_md5sum) = @_; my $found = 0; # What are my IPs? my $local_host = $anvil->Get->short_host_name(); $anvil->Network->get_ips(); foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$local_host}{interface}}) { next if not $anvil->data->{network}{$local_host}{interface}{$interface}{ip}; next if not $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask}; my $ip = $anvil->data->{network}{$local_host}{interface}{$interface}{ip}; my $subnet_mask = $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask}; my $network = $anvil->Network->get_network({ip => $ip, subnet_mask => $subnet_mask}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 's1:interface' => $interface, 's2:ip' => $ip, 's3:subnet_mask' => $subnet_mask, 's4:network' => $network, }}); my $type = "other"; my $sort = $interface; if ($interface =~ /^((?:bc|s|if)n)(\d+)/) { $type = $1; $sort = $2; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 's1:type' => $type, 's2:sort' => $sort, }}); } $anvil->data->{local_ip}{by_network}{$network}{type} = $type; $anvil->data->{local_ip}{by_network}{$network}{'sort'} = $sort; $anvil->data->{local_ip}{by_network}{$network}{interface} = $interface; $anvil->data->{local_ip}{by_network}{$network}{ip} = $ip; $anvil->data->{local_ip}{by_network}{$network}{subnet_mask} = $subnet_mask; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "s1:local_ip::by_network::${network}::type" => $anvil->data->{local_ip}{by_network}{$network}{type}, "s1:local_ip::by_network::${network}::sort" => $anvil->data->{local_ip}{by_network}{$network}{'sort'}, "s2:local_ip::by_network::${network}::interface" => $anvil->data->{local_ip}{by_network}{$network}{interface}, "s3:local_ip::by_network::${network}::ip" => $anvil->data->{local_ip}{by_network}{$network}{ip}, "s4:local_ip::by_network::${network}::subnet_mask" => $anvil->data->{local_ip}{by_network}{$network}{subnet_mask}, }}); } # Create a hash we can sort through that are on the same subnet_mask as us. my $query = " SELECT a.host_uuid, a.host_name, a.host_type, b.file_directory, b.file_name, b.file_size, b.file_md5sum FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid AND b.file_uuid = ".$anvil->Database->quote($file_uuid)." AND a.host_uuid != ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." ORDER BY file_mtime DESC; ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { results => $results, count => $count, }}); foreach my $row (@{$results}) { my $host_uuid = $row->[0]; my $host_name = $row->[1]; my $host_type = $row->[2]; my $file_directory = $row->[3]; my $file_name = $row->[4]; my $file_size = $row->[5]; my $file_md5sum = $row->[6]; my $test_file = $file_directory."/".$file_name; $test_file =~ s/\/\//\//g; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_uuid => $host_uuid, host_name => $host_name, host_type => $host_type, file_directory => $file_directory, file_name => $file_name, file_size => $file_size, file_md5sum => $file_md5sum, test_file => $test_file, }}); # What IP addresses are on this machine? my $query = " SELECT ip_address_address, ip_address_subnet_mask FROM ip_addresses WHERE ip_address_note != 'DELETED' AND ip_address_host_uuid = ".$anvil->Database->quote($host_uuid)." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { results => $results, count => $count, }}); foreach my $row (@{$results}) { my $ip_address_address = $row->[0]; my $ip_address_subnet_mask = $row->[1]; my $network = $anvil->Network->get_network({ip => $ip_address_address, subnet_mask => $ip_address_subnet_mask}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { ip_address_address => $ip_address_address, ip_address_subnet_mask => $ip_address_subnet_mask, network => $network, }}); # Are we on the same subnet? if (exists $anvil->data->{local_ip}{by_network}{$network}) { # We're on the same subnet! my $type = $anvil->data->{local_ip}{by_network}{$network}{type}; my $sort = $anvil->data->{local_ip}{by_network}{$network}{'sort'}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "s1:type" => $type, "s2:sort" => $sort, }}); # Record. $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip} = $ip_address_address; $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name} = $host_name; $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid} = $host_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "s1:peer_ip::${host_type}::${type}::${sort}::name" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name}, "s2:peer_ip::${host_type}::${type}::${sort}::ip" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip}, "s3:peer_ip::${host_type}::${type}::${sort}::host_uuid" => $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid}, }}); } } } ### TODO: Do this according to what we are. # Sort through what we've found. my $file_found = 0; my $searched_hosts = {}; foreach my $host_type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}}) { last if $file_found; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }}); foreach my $type (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}}) { last if $file_found; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }}); foreach my $sort (sort {$a cmp $b} keys %{$anvil->data->{peer_ip}{$host_type}{$type}}) { last if $file_found; my $ip = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{ip}; my $name = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{name}; my $host_uuid = $anvil->data->{peer_ip}{$host_type}{$type}{$sort}{host_uuid}; my $remote_user = "admin"; my $password = ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:host_type' => $host_type, 's2:type' => $type, 's3:sort' => $sort, 's4:name' => $name, 's5:ip' => $ip, 's6:host_uuid' => $host_uuid, 's7:remote_user' => $remote_user, 's8:password' => $password, }}); if ((exists $searched_hosts->{$name}) && ($searched_hosts->{$name})) { # Already searched this host, this is just a different IP. Skip it. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0282", variables => { host_name => $name, ip => $ip, }}); next; } $searched_hosts->{$name} = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "searched_hosts->{$name}" => $searched_hosts->{$name} }}); ### NOTE: There's a bug in Net::SSH2 on RHEL8 where passwordless SSH doesn't ### work. So for now, we'll manually pull in passwords from the ### anvil.conf using 'hosts::::password' or ### 'hosts::::password'. This will be removed when the bug ### is fixed. if ((exists $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}) && ($anvil->data->{hosts}{$host_uuid}{password}{$remote_user})) { $password = $anvil->data->{hosts}{$host_uuid}{password}{$remote_user}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }}); } elsif ((exists $anvil->data->{hosts}{$name}{password}{$remote_user}) && ($anvil->data->{hosts}{$name}{password}{$remote_user})) { $password = $anvil->data->{hosts}{$host_uuid}{password}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { password => $password }}); } ### NOTE: This might take a while for the call to return if we're md5sum'ing ### a large file like an ISO. ### TODO: Should we ask for the size, then follow up with the md5sum for ### files that are over a certain size? Or will this get called rarely ### enough in practice that it doesn't really matter? # If the file is found, we'll parse these out. my $remote_size = 0; my $remote_md5sum = ""; my ($output, $error, $return_code) = $anvil->Remote->call({ shell_call => $anvil->data->{path}{exe}{'anvil-file-details'}." --file ".$full_path." --with-md5sum".$anvil->Log->switches, remote_user => $remote_user, password => $password, target => $ip, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { error => $error, output => $output, return_code => $return_code, }}); foreach my $line (split/\n/, $output) { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }}); if ($line =~ /^size: \[(\d+)\]$/) { $remote_size = $1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_size => $remote_size }}); } if ($line =~ /^md5sum: \[(.*)\]$/) { $remote_md5sum = $1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { remote_md5sum => $remote_md5sum }}); } } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:remote_size' => $remote_size, 's2:file_size' => $file_size, 's3:remote_md5sum' => $remote_md5sum, 's4:file_md5sum' => $file_md5sum, }}); ### Do I really need to match sizes if the md5sum is the same? if (($remote_size eq $file_size) && ($remote_md5sum eq $file_md5sum)) { # Pull it over! $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0276", variables => { file => $full_path, host_name => $name, ip => $ip, }}); my $failed = $anvil->Storage->rsync({ debug => 2, destination => $full_path, password => $password, source => $remote_user."\@".$ip.":".$full_path, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); if (-f $full_path) { # Got it! $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0277", variables => { file => $full_path }}); # Verify the md5sum. my $local_md5sum = $anvil->Get->md5sum({file => $full_path}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_md5sum => $local_md5sum, file_md5sum => $file_md5sum, }}); if ($local_md5sum eq $file_md5sum) { # Success! $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0278", variables => { file => $full_path }}); $file_found = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_found => $file_found }}); last; } else { # Failed. :( $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0279", variables => { file => $full_path }}); unlink $full_path; } } else { # Failed to rsync. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0280", variables => { file => $full_path, host_name => $name, ip => $ip, }}); } } else { # Doesn't match what we're looking for. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0281", variables => { file => $full_path, host_name => $name, ip => $ip, remote_md5sum => $remote_md5sum, file_md5sum => $file_md5sum, file_size => $file_size, say_file_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}), remote_size => $remote_size, say_remote_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $remote_size}), }}); } } } } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); return($found); } # This handles deleting a file. sub check_incoming { my ($anvil) = @_; # This hash shouldn't exist, but lets be safe... if (exists $anvil->data->{scan}{directories}) { delete $anvil->data->{scan}{directories}; } # Read any files in '/mnt/shared/incoming'. $anvil->Storage->scan_directory({ debug => 3, directory => $anvil->data->{path}{directories}{shared}{base}, recursive => 1, }); my $incoming_directory = $anvil->data->{path}{directories}{shared}{incoming}; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0264"}); foreach my $full_path (sort {$a cmp $b} keys %{$anvil->data->{scan}{directories}}) { # Skip this if it's under '/mnt/shared/temp' (that's used for files being downloaded, etc) next if $full_path =~ /^\/mnt\/shared\/temp\//; # Skip if this isn't a file. my $file_type = $anvil->data->{scan}{directories}{$full_path}{type}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { full_path => $full_path, file_type => $file_type, }}); next if $file_type ne "file"; my $file_name = $anvil->data->{scan}{directories}{$full_path}{name}; my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory}; my $file_size = $anvil->data->{scan}{directories}{$full_path}{size}; my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime}; my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; my $file_executable = $anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0; my $say_mimetype = convert_mimetype($anvil, $file_mimetype, $full_path, $file_executable); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_name => $file_name, file_size => $file_size, file_mtime => $file_mtime, file_mimetype => $file_mimetype, file_executable => $file_executable, say_mimetype => $say_mimetype, }}); # Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum. my ($file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = get_file_db_info($anvil, "", $file_name); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid, recorded_size => $recorded_size, recorded_mtime => $recorded_mtime, recorded_md5sum => $recorded_md5sum, }}); # Calculate the md5sum? my $file_md5sum = $recorded_md5sum; if ((not $file_uuid) or ($file_size != $recorded_size)) { # Yes. But first, do we have a size mismatch? If so, see if we need to pull a newer # version down from elsewhere. if (($file_uuid) && ($file_mtime <= $recorded_mtime)) { # We've got an older file, we need to update. pull_file($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum); # TODO: Now see if it exists and, if it does, re-stat it. If not, loop to the # next file and skip this one. } # Now generate the md5sum. If this file is over 128 MiB, warn the user that it might # take a while. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0265", variables => { file => $full_path }}); if ($file_size > (128 * (2 ** 20))) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0266", variables => { size => $anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}), }}); } # Update (or get) the md5sum. $file_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype, file_md5sum => $file_md5sum, }}); # Insert or update the files entry. ($file_uuid) = $anvil->Database->insert_or_update_files({ debug => 3, file_uuid => $file_uuid, file_name => $file_name, file_directory => $file_directory, file_size => $file_size, file_md5sum => $file_md5sum, file_mtime => $file_mtime, file_type => $say_mimetype, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); } # If we still don't have a file UUID for some reason, skip this file. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); next if not $file_uuid; # Make sure we know about this file on this system my ($file_locatiom_uuid) = $anvil->Database->insert_or_update_file_locations({ debug => 3, file_location_file_uuid => $file_uuid, file_location_host_uuid => $anvil->data->{sys}{host_uuid}, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_locatiom_uuid => $file_locatiom_uuid }}); # Are we in the incoming directory? If so, move the file. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { full_path => $full_path, incoming_directory => $incoming_directory, }}); if ($full_path =~ /^$incoming_directory/) { # If it's a definition file, we'll move it to # 'path::directories::shared::definitions', otherwise we'll move it to # 'path::directories::shared::files'. my $target = $say_mimetype eq "definition" ? $anvil->data->{path}{directories}{shared}{definitions} : $anvil->data->{path}{directories}{shared}{files}; $target .= "/"; $target =~ s/\/\//\//g; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { target => $target }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0268", variables => { file => $full_path, target => $target, }}); $anvil->Storage->move_file({ debug => 3, source_file => $full_path, target_file => $target, }); # Update the file_directory. my $say_directory =~ s/\/$//; ($file_uuid) = $anvil->Database->insert_or_update_files({ debug => 3, file_uuid => $file_uuid, file_name => $file_name, file_directory => $say_directory, file_size => $file_size, file_md5sum => $file_md5sum, file_mtime => $file_mtime, file_type => $say_mimetype, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); } } return(0); } # This method finds a file elsewhere on the network and pulls it to here. sub pull_file { my ($anvil, $file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = @_; $file_uuid = "" if not defined $file_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid, recorded_size => $recorded_size, recorded_mtime => $recorded_mtime, recorded_md5sum => $recorded_md5sum, }}); # Find the hosts with this file, then connect to it to see if the size is the same as what we want. # If so, pull it down... ### TODO return(0); } # This gets the file_uuid for a given file name and/or md5sum. If the file isn't found, an empty string is # returned. If it is found, the file size as recorded in the database is returned. sub get_file_db_info { my ($anvil, $file_md5sum, $file_name) = @_; $file_md5sum = "" if not defined $file_md5sum; $file_name = "" if not defined $file_name; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_md5sum => $file_md5sum, file_name => $file_name, }}); # Get the file size and file uuid, if possible. # If I have the md5sum, search using that. If I have the filename only, then we'll fall back to that. my $query = " SELECT file_uuid, file_size, file_mtime, file_md5sum FROM files WHERE "; if ($file_md5sum) { $query .= " file_md5sum = ".$anvil->Database->quote($file_md5sum)."\n"; } elsif ($file_name) { $query .= " file_name = ".$anvil->Database->quote($file_name)."\n"; } my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { results => $results, count => $count, }}); if (not $count) { # File wasn't found in the database return("", 0, 0, ""); } my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : ""; my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : 0; my $file_mtime = defined $results->[0]->[2] ? $results->[0]->[2] : 0; $file_md5sum = defined $results->[0]->[3] ? $results->[0]->[3] : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid, file_size => $file_size, file_mtime => $file_mtime, file_md5sum => $file_md5sum, }}); return($file_uuid, $file_size, $file_mtime, $file_md5sum); } # This handles toggling a file to marked or unmarked as a script. sub handle_script { my ($anvil) = @_; if (not $anvil->data->{switches}{file}) { # Um... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0055"}); $anvil->Job->update_progress({ progress => 0, message => "error_0055", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 3}); } # Find the file_uuid (we don't actually care about the file size, mtime or md5sum). my ($file_uuid, $file_size, $recorded_mtime, $file_md5sum) = get_file_db_info($anvil, "", $anvil->data->{switches}{file}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); # Toggle the executable bits. my $executable = 0; if ($anvil->data->{switches}{'is-script'}) { # Is it already executable? if (-x $anvil->data->{switches}{file}) { # Switch it on $executable = 1; $anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a+x"}); } else { # Already a script. } } else { # Is it executable? if (-x $anvil->data->{switches}{file}) { # Switch it off. $executable = 1; $anvil->Storage->change_mode({path => $anvil->data->{switches}{file}, mode => "a-x"}); } else { # Already not a script. } } # If we have a file UUID, update the 'file_type if ($file_uuid) { # Load the details. my $query = " SELECT file_name, file_directory, file_size, file_md5sum, file_type, file_mtime FROM files WHERE file_uuid = ".$anvil->Database->quote($file_uuid)." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }}); my $file_name = $results->[0]->[0]; my $file_directory = $results->[0]->[1]; my $file_size = $results->[0]->[2]; my $file_md5sum = $results->[0]->[3]; my $file_type = $results->[0]->[4]; my $file_mtime = $results->[0]->[5]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_name => $file_name, file_directory => $file_directory, file_size => $file_size, file_md5sum => $file_md5sum, file_type => $file_type, file_mtime => $file_mtime, }}); if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'})) { # Figure out what the file type is and update. my $mimetype = mimetype($file_name); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mimetype => $mimetype }}); my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }}); $anvil->Database->insert_or_update_files({ debug => 2, file_uuid => $file_uuid, file_name => $anvil->data->{switches}{file}, file_directory => $file_directory, file_size => $file_size, file_md5sum => $file_md5sum, file_mtime => $file_mtime, file_type => $say_mimetype, }); } elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'})) { # Change the file tpye to "script" $anvil->Database->insert_or_update_files({ debug => 2, file_uuid => $file_uuid, file_name => $anvil->data->{switches}{file}, file_directory => $file_directory, file_size => $file_size, file_md5sum => $file_md5sum, file_mtime => $file_mtime, file_type => "script", }); } } return(0); } # This takes the mimetype as reported by the 'mimetype' method and returns a file type we care about. sub convert_mimetype { my ($anvil, $mimetype, $file, $executable) = @_; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { mimetype => $mimetype, file => $file, executable => $executable, }}); my $say_mimetype = "other"; if ($mimetype) { if ($mimetype =~ /cd-image/) { $say_mimetype = "iso"; } elsif ($mimetype =~ /xml/) { # This might be a definition, but look inside it to be sure. my ($is_domain, $return_code) = $anvil->System->call({debug => 3, shell_call => "if \$(".$anvil->data->{path}{exe}{'grep'}." -q '' ".$file."); then ".$anvil->data->{path}{exe}{echo}." 1; else ".$anvil->data->{path}{exe}{echo}." 0; fi" }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { is_domain => $is_domain, return_code => $return_code }}); if ($is_domain) { $say_mimetype = "definition"; } } elsif ($mimetype =~ /rpm$/) { $say_mimetype = "rpm"; } elsif ($mimetype =~ /disk-image/) { $say_mimetype = "disk-image"; } elsif ($executable) { $say_mimetype = "script"; } } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_mimetype => $say_mimetype }}); return($say_mimetype); } # This looks for any files with the file_type of 'DELETED' and that has an entry in file_locations for this # machine. Any that are found will have their file deleted and the file_locations entry deleted. sub check_for_deletes { my ($anvil) = @_; my $query = " SELECT a.file_uuid, b.file_location_uuid, a.file_directory || '/' || a.file_name AS full_path FROM files a, file_locations b WHERE a.file_uuid = b.file_location_file_uuid AND a.file_type = 'DELETED' AND b.file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { results => $results, count => $count, }}); foreach my $row (@{$results}) { my $file_uuid = $row->[0]; my $file_location_uuid = $row->[1]; my $full_path = $row->[2]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 's1:file_uuid' => $file_uuid, 's2:file_location_uuid' => $file_location_uuid, 's3:full_path' => $full_path, }}); # Get rid of it (if it actually exists) and then remove it from file_locations. if (-e $full_path) { # Delete it. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0285", variables => { file => $full_path }}); unlink $full_path; # Sleep and verify sleep 1; if (-e $full_path) { # Failed to delete... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284", variables => { file => $full_path }}); return(""); } else { # Deleted successfully. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"}); } } # Delete the entry from file_locations, if needed. $query = "DELETE FROM file_locations WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid).";"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } return(0); } # This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the # database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a # file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts. sub handle_delete { my ($anvil) = @_; my $full_path = $anvil->data->{switches}{file}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }}); if (not $full_path) { # Um... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"}); $anvil->Job->update_progress({ progress => 0, message => "error_0052", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 3}); } elsif ($full_path !~ /^\/mnt\/shared\//) { # We don't do that here... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $full_path }}); $anvil->Job->update_progress({ progress => 0, message => "error_0053,!!file!".$full_path."!!", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 6}); } # What's the UUID of this file? I collect all the data for the update, if appropriate. my $file_name = ($full_path =~ /^.*\/(.*)$/)[0]; my $file_directory = ($full_path =~ /^(.*?)\/$file_name$/)[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_name => $file_name, file_directory => $file_directory, }}); my $query = " SELECT file_uuid, file_size, file_md5sum, file_type, file_mtime FROM files WHERE file_name = ".$anvil->Database->quote($file_name)." AND file_directory = ".$anvil->Database->quote($file_directory)." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results, count => $count, }}); my $file_uuid = defined $results->[0]->[0] ? $results->[0]->[0] : ""; my $file_size = defined $results->[0]->[1] ? $results->[0]->[1] : ""; my $file_md5sum = defined $results->[0]->[2] ? $results->[0]->[2] : ""; my $file_type = defined $results->[0]->[3] ? $results->[0]->[3] : ""; my $file_mtime = defined $results->[0]->[4] ? $results->[0]->[4] : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid, file_size => $file_size, file_md5sum => $file_md5sum, file_type => $file_type, file_mtime => $file_mtime, }}); # If I have the file_uuid, and we've been asked to delete it everywhere, mark it as DELETEd in the # database. This way, if anything goes wrong below, future runs will try (again) to delete the file # itself. If it's only beind deleted locally, and it fails for some reason, there will be no attempt # to try again. if (($file_uuid) && ($anvil->data->{switches}{everywhere})) { # Yup. ($file_uuid) = $anvil->Database->insert_or_update_files({ file_uuid => $file_uuid, file_name => $file_name, file_directory => $file_directory, file_size => $file_size, file_md5sum => $file_md5sum, file_type => "DELETED", file_mtime => $file_mtime, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { file_uuid => $file_uuid }}); } # Does the file exist? if (-e $full_path) { # Delete it. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0263", variables => { file => $full_path }}); unlink $full_path; # Sleep and verify sleep 1; if (-e $full_path) { # Failed to delete... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0284"}); return(""); } else { # Deleted successfully. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0283"}); } } # Now, if I have a file_uuid, delete it from file_locations if it exists. if ($file_uuid) { # Delete the entry from file_locations, if needed. my $query = " DELETE FROM file_locations WHERE file_location_file_uuid = ".$anvil->Database->quote($file_uuid)." AND file_location_host_uuid = ".$anvil->Database->quote($anvil->data->{sys}{host_uuid})." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } return(0); } # This handles renaming files. sub handle_rename { my ($anvil) = @_; # Do we have the current file name and the new one? if (not $anvil->data->{switches}{file}) { # Um... $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0049"}); $anvil->Job->update_progress({ progress => 0, message => "error_0049", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 3}); } elsif (not $anvil->data->{switches}{to}) { # We need a target $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0050", variables => { file => $anvil->data->{switches}{file} }}); $anvil->Job->update_progress({ progress => 0, message => "error_0050,!!file!".$anvil->data->{switches}{file}."!!", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 4}); } elsif (not -f $anvil->data->{switches}{file}) { # The file to rename doesn't exist. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0051", variables => { file => $anvil->data->{switches}{file} }}); $anvil->Job->update_progress({ progress => 0, message => "error_0051,!!file!".$anvil->data->{switches}{file}."!!", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 5}); } elsif (-e $anvil->data->{switches}{to}) { # There's already a file (or directory or something) with that name. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0056", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }}); $anvil->Job->update_progress({ progress => 0, message => "error_0056,!!file!".$anvil->data->{switches}{file}."!!,!!to!".$anvil->data->{switches}{to}."!!", job_uuid => $anvil->data->{jobs}{'job-uuid'}, }); $anvil->nice_exit({exit_code => 6}); } ### TODO: Left off here return(0); }