#!/usr/bin/perl # # This runs on striker dashboards and syncs files under /mnt/shared on all known systems. It reaches out and # pulls over any files under /mnt/shared/files/ to the same on the local system. It then pushes files out to # all members of the same Anvil!. # # If this is called with a job-uuid, file-specific tasks will be handled, like moving files uploaded over a # browser or deleting / purging a file. # # NOTE: This file is NOT responsible for sync'ing definition files! That is handles in scan-server. # # TODO: # - Handle deleting files by user input, or if a given file that was on an Anvil! has been removed for both # nodes and DR, where applicable. # - use strict; use warnings; use Anvil::Tools; use Data::Dumper; my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; if (($running_directory =~ /^\./) && ($ENV{PWD})) { $running_directory =~ s/^\./$ENV{PWD}/; } # Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. $| = 1; my $anvil = Anvil::Tools->new(); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); # Read switches (target ([user@]host[:port]) and the file with the target's password. $anvil->data->{switches}{'job-uuid'} = ""; $anvil->Get->switches; # Connect to the database(s). $anvil->Database->connect; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); # If we don't have a job-uuid, look for one. if (not $anvil->data->{switches}{'job-uuid'}) { # Load the job data. $anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({debug => 2, program => $THIS_FILE}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }}); } # If we still don't have a job-uuit, go into interactive mode. if ($anvil->data->{switches}{'job-uuid'}) { # Load the job data. $anvil->Job->get_job_details({debug => 2}); $anvil->Job->clear(); $anvil->data->{sys}{progress} = 1; if ($anvil->data->{jobs}{job_name} eq "upload::move_incoming") { process_incoming_file($anvil); } # Job data will be in $anvil->data->{jobs}{job_data} } else { # Do a normal periodic search. ### NOTE: When finding new files, check the size, sleep for 30 seconds, and check again. If a file's ### size changed, skip it, it's likely still being updated. } $anvil->nice_exit({exit_code => 0}); ############################################################################################################# # Functions # ############################################################################################################# sub process_incoming_file { my ($anvil) = @_; $anvil->Job->update_progress({ progress => $anvil->data->{sys}{progress}, message => "message_0191", }); my $file = ($anvil->data->{jobs}{job_data} =~ /file=(.*)$/)[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }}); if (not $file) { # Can't do anything, file wasn't parsed. $anvil->Job->update_progress({ progress => 100, message => "error_0170,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", job_status => "failed", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0170", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }}); $anvil->nice_exit({exit_code => 1}); } elsif (not -e $file) { # Can't do anything, file doesn't exist $anvil->Job->update_progress({ progress => 100, message => "error_0171,!!file!".$file."!!", job_status => "failed", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }}); $anvil->nice_exit({exit_code => 1}); } # Move it over to files. $anvil->data->{sys}{progress} = 10; $anvil->Job->update_progress({ progress => $anvil->data->{sys}{progress}, message => "message_0192,!!file!".$file."!!", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0192", variables => { file => $file }}); $anvil->Storage->move_file({ debug => 2, overwrite => 1, source_file => $file, target_file => $anvil->data->{path}{directories}{shared}{files}."/", }); my $file_name = ($file =~ /\/.*\/(.*?)$/)[0]; my $target_file = $anvil->data->{path}{directories}{shared}{files}."/".$file_name; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_name => $file_name, target_file => $target_file, }}); if (not -e $target_file) { # Failed to move. $anvil->Job->update_progress({ progress => 100, message => "error_0172,!!file!".$file."!!,!!target_directory!".$anvil->data->{path}{directories}{shared}{files}."!!", job_status => "failed", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }}); $anvil->nice_exit({exit_code => 1}); } # Calculate the md5sum. $anvil->data->{sys}{progress} = 20; $anvil->Job->update_progress({ progress => $anvil->data->{sys}{progress}, message => "message_0193", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0193"}); my ($string, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{md5sum}." ".$target_file}); my $md5sum = ($string =~ /^(.*?)\s/)[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { string => $string, md5sum => $md5sum, return_code => $return_code, }}); # Store the file details! $anvil->data->{sys}{progress} = 30; $anvil->Job->update_progress({ progress => $anvil->data->{sys}{progress}, message => "message_0194,!!md5sum!".$md5sum."!!", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0194", variables => { md5sum => $md5sum }}); $anvil->Storage->get_file_stats({ debug => 2, file_path => $target_file, }); my $file_mimetype = $anvil->data->{file_stat}{$target_file}{mimetype}; my $file_size = $anvil->data->{file_stat}{$target_file}{size}; my $file_mtime = $anvil->data->{file_stat}{$target_file}{modified_time}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_mimetype => $file_mimetype, file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")", file_mtime => $file_mtime, }}); # This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or # 'other'. If set to 'DELETED', the file will be removed from disk. my $file_type = "other"; if ($file_mimetype =~ /cd-image/) { $file_type = "iso"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_type => $file_type }}); } my $file_uuid = $anvil->Database->insert_or_update_files({ debug => 2, file_name => $file_name, file_directory => $anvil->data->{path}{directories}{shared}{files}, file_size => $file_size, file_md5sum => $md5sum, file_type => $file_type, file_mtime => $file_mtime, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); # Now copy this to our peers. foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}}) { # Periodically, autovivication causes and empty key to appear. $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid => $uuid }}); next if ((not $uuid) or (not $anvil->Validate->uuid({uuid => $uuid}))); next if $uuid eq $anvil->Get->host_uuid; my $host = $anvil->data->{database}{$uuid}{host}; my $password = $anvil->data->{database}{$uuid}{password}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host => $host, password => $anvil->Log->is_secure($password), }}); my $striker_name = $anvil->Get->host_name_from_uuid({host_uuid => $uuid}); my $say_host = $striker_name." (".$host.")"; # Rsync the file. $anvil->data->{sys}{progress} += 10; $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; $anvil->Job->update_progress({ progress => $anvil->data->{sys}{progress}, message => "message_0195,!!host!".$say_host."!!", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => { host => $say_host }}); $anvil->Storage->rsync({ debug => 2, source => $target_file, destination => "root\@".$host.":".$anvil->data->{path}{directories}{shared}{files}."/", try_again => 1, }); } ### TODO: Make is an upload-time option to choose if the uploaded file automatically goes to any given Anvil! # Tell other Anvil! systems to download this file. $anvil->Database->get_anvils({debug => 2}); foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}}) { my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid}; my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node1_host_uuid}; my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node2_host_uuid}; my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid, anvil_node1_host_uuid => $anvil_node1_host_uuid, anvil_node2_host_uuid => $anvil_node2_host_uuid, anvil_dr1_host_uuid => $anvil_dr1_host_uuid, }}); $anvil->data->{sys}{progress} += 5; $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; $anvil->Job->update_progress({ progress => $anvil->data->{sys}{progress}, message => "message_0196,!!anvil_name!".$anvil_name."!!", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => { anvil_name => $anvil_name }}); my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({ debug => 2, file_location_file_uuid => $file_uuid, file_location_anvil_uuid => $anvil_uuid, file_location_active => 1, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); ### TODO: Register a job for each member to run this file with an 'upload::pull_file' with ### the job_data being the 'file=$target_file'. Have node1 try the first striker, and ### node2 from the second striker, if it exists. DR should watch node1 and node2's ### jobs and not download until those jobs hit 100. } # Done! $anvil->Job->update_progress({ progress => 100, message => "message_0197", }); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0197"}); return(0); }