anvil/tools/anvil-sync-shared

#!/usr/bin/perl
# 
# This runs on striker dashboards and syncs files under /mnt/shared on all known systems. It reaches out and 
# pulls over any files under /mnt/shared/files/ to the same on the local system. It then pushes files out to
# all members of the same Anvil!. 
# 
# If this is called with a job-uuid, file-specific tasks will be handled, like moving files uploaded over a
# browser or deleting / purging a file.
# 
# NOTE: This file is NOT responsible for sync'ing definition files! That is handles in scan-server.
# 
# TODO:
# - Handle deleting files by user input, or if a given file that was on an Anvil! has been removed for both
#   nodes and DR, where applicable.
# - 
 
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();

# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->Get->switches;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

# Connect to the database(s).
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});

# If we don't have a job-uuid, look for one.
if (not $anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({debug => 2, program => $THIS_FILE});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
}

# If we still don't have a job-uuid, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->get_job_details({debug => 3});
	$anvil->Job->clear({debug => 3});
	
	$anvil->data->{sys}{progress} = 1;
	if ($anvil->data->{jobs}{job_name} eq "storage::move_incoming")
	{
		process_incoming_file($anvil);
	}
	if ($anvil->data->{jobs}{job_name} eq "storage::pull_file")
	{
		process_pull_file($anvil);
	}
	if ($anvil->data->{jobs}{job_name} eq "storage::purge")
	{
		process_purge_file($anvil);
	}
	if ($anvil->data->{jobs}{job_name} eq "storage::rename")
	{
		process_rename_file($anvil);
	}
	if ($anvil->data->{jobs}{job_name} eq "storage::check_mode")
	{
		process_file_mode($anvil);
	}
}
else
{
	# This checks to see if there are any files that need to be pulled, renamed, removed, or mode 
	# updated.
	#run_checks($anvil);
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

### TODO: Left off here. Run periodic checks for files that need to be pulled/removed, mode changed,
###       etc. For 3.0, we'll ignore files people add manually though later we'll probably want to 
###       auto-sync them.
### NOTE: When finding new files, check the size, sleep for 30 seconds, and check again. If a file's 
###       size changed, skip it, it's likely still being updated.
### NOTE: This isn't finished, and likely won't be for a while.
sub run_checks
{
	my ($anvil) = @_;
	
	# First, get a list of files we're expected to have.
	$anvil->Database->get_files({debug => 3, include_deleted => 1});
	$anvil->Database->get_file_locations({debug => 3});
	
	if ($anvil->Get->host_type eq "striker")
	{
		# Look for files to add
	}
	else
	{
		# Are we an Anvil! member?
		my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
		
		if (not $anvil_uuid)
		{
			# Nothing to do.
			$anvil->nice_exit({exit_code => 0});
		}
		
		# What files should we have?
		foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{files}{file_name}})
		{
			# TODO: Once per day, calculate the md5sums and compare against the DB.
			my $file_uuid            = $anvil->data->{files}{file_name}{$file_name}{file_uuid};
			my $file_directory       = $anvil->data->{files}{file_name}{$file_name}{file_directory};
			my $file_size            = $anvil->data->{files}{file_name}{$file_name}{file_size};
			my $file_type            = $anvil->data->{files}{file_name}{$file_name}{file_type};
			my $file_path            = $file_directory."/".$file_name;
			my $file_location_uuid   = $anvil->data->{file_locations}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
			my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				file_uuid            => $file_uuid, 
				file_name            => $file_name,
				file_directory       => $file_directory, 
				file_size            => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
				file_type            => $file_type, 
				file_path            => $file_path, 
				file_location_uuid   => $file_location_uuid, 
				file_location_active => $file_location_active, 
			}});
			
			if (-e $file_path)
			{
				# File exists, should it?
			}
			else
			{
				# File doesn't exist, should it?
			}
		}
	}
	
	return(0);
}

sub process_file_mode
{
	my ($anvil) = @_;
	
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress},
		message  => "message_0205", 
	});
	
	# We have to include deleted because purges will already have the file flagged as deleted.
	$anvil->Database->get_files({debug => 3, include_deleted => 1});
	$anvil->Database->get_file_locations({debug => 3});
	
	my $file_uuid = ($anvil->data->{jobs}{job_data} =~ /file_uuid=(.*)$/)[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
	
	if (not $file_uuid)
	{
		# Can't do anything, file wasn't parsed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0185,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0185", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	elsif (not exists $anvil->data->{files}{file_uuid}{$file_uuid})
	{
		# File UUID doesn't appear to be valid.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0186,!!file_uuid!".$file_uuid."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0186", variables => { file_uuid => $file_uuid }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	my $file_name      = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
	my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
	my $file_path      = $file_directory."/".$file_name;
	my $file_type      = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:file_name'      => $file_name, 
		's2:file_directory' => $file_directory, 
		's3:file_path'      => $file_path, 
		's4:file_type'      => $file_type, 
	}});
	
	# From here, we'll unlink if it exists. If it doesn't exist, we still exit sucessfully.
	if (-e $file_path)
	{
		# Change the mode, if need it.
		my $new_mode = "a-x";
		if ($file_type eq "script")
		{
			$new_mode = "a+x";
		}
		$anvil->Storage->change_mode({
			path => $file_path, 
			mode => $new_mode,
		});
		$anvil->Job->update_progress({
			progress => 100,
			message  => "job_0145,!!file_path!".$file_path."!!,!!new_mode!".$new_mode."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0145", variables => { 
			file_path => $file_path,
			new_mode  => $new_mode, 
		}});
		$anvil->nice_exit({exit_code => 0});
	}
	else
	{
		# Doesn't exist, nothing to do.
		$anvil->Job->update_progress({
			progress => 100,
			message  => "job_0146,!!file_path!".$file_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0146", variables => { file_path => $file_path }});
		$anvil->nice_exit({exit_code => 0});
	}
	
	return(0);
}

sub process_rename_file
{
	my ($anvil) = @_;
	
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress},
		message  => "message_0204", 
	});

	$anvil->Database->get_files({debug => 3});
	$anvil->Database->get_file_locations({debug => 3});
	
	my $file_uuid = "";
	my $old_name  = "";
	my $new_name  = "";
	foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /file_uuid=(.*)$/)
		{
			$file_uuid = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
		}
		if ($line =~ /old_name=(.*)$/)
		{
			$old_name = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_name => $old_name }});
		}
		if ($line =~ /new_name=(.*)$/)
		{
			$new_name = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_name => $new_name }});
		}
	}
	
	if (not $file_uuid)
	{
		# Can't do anything, file wasn't parsed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0181,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0181", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	elsif (not exists $anvil->data->{files}{file_uuid}{$file_uuid})
	{
		# File UUID doesn't appear to be valid.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0182,!!file_uuid!".$file_uuid."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0182", variables => { file_uuid => $file_uuid }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	my $file_name      = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
	my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
	my $old_file_path  = $file_directory."/".$old_name;
	my $new_file_path  = $file_directory."/".$new_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:file_name'      => $file_name, 
		's2:file_directory' => $file_directory, 
		's3:old_file_path'  => $old_file_path, 
		's4:new_file_path'  => $new_file_path, 
	}});
	
	if (not $new_name)
	{
		# Can't do anything, file wasn't parsed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0183,!!file_name!".$new_file_path."!!,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0183", variables => { 
			file_name => $new_file_path, 
			job_uuid  => $anvil->data->{switches}{'job-uuid'},
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# If the file doesn't exist, there's nothing to do (and that's OK).
	if (-e $old_file_path)
	{
		# Move the file.
		$anvil->Job->update_progress({
			progress => 50,
			message  => "job_0141,!!old_file!".$old_file_path."!!,!!new_file!".$new_file_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0141", variables => { 
			old_file => $old_file_path, 
			new_file => $new_file_path, 
		}});
		$anvil->Storage->move_file({
			source_file => $old_file_path, 
			target_file => $new_file_path,
		});
		
		if ((not -e $old_file_path) && (-e $new_file_path))
		{
			# Success!
			$anvil->Job->update_progress({
				progress => 100,
				message  => "job_0142", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0142"});
			$anvil->nice_exit({exit_code => 0});
		}
		elsif ((-e $old_file_path) && (not -e $new_file_path))
		{
			# Move failed
			$anvil->Job->update_progress({
				progress   => 1,
				message    => "error_0184", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0184"});
			sleep 60;
			$anvil->nice_exit({exit_code => 1});
		}
	}
	else
	{
		# Nothing to do
		$anvil->Job->update_progress({
			progress => 100,
			message  => "job_0140,!!file_path!".$old_file_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0140", variables => { file_path => $old_file_path }});
		$anvil->nice_exit({exit_code => 0});
	}
	
	return(0);
}

sub process_purge_file
{
	my ($anvil) = @_;
	
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress},
		message  => "message_0203", 
	});
	
	# We have to include deleted because purges will already have the file flagged as deleted.
	$anvil->Database->get_files({debug => 3, include_deleted => 1});
	$anvil->Database->get_file_locations({debug => 3});
	
	my $file_uuid = ($anvil->data->{jobs}{job_data} =~ /file_uuid=(.*)$/)[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
	
	if (not $file_uuid)
	{
		# Can't do anything, file wasn't parsed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0178,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0178", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	elsif (not exists $anvil->data->{files}{file_uuid}{$file_uuid})
	{
		# File UUID doesn't appear to be valid.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0179,!!file_uuid!".$file_uuid."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0179", variables => { file_uuid => $file_uuid }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	my $file_name      = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
	my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
	my $file_path      = $file_directory."/".$file_name;
	my $file_type      = $anvil->data->{files}{file_uuid}{$file_uuid}{file_type};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:file_name'      => $file_name, 
		's2:file_directory' => $file_directory, 
		's3:file_path'      => $file_path, 
		's4:file_type'      => $file_type, 
	}});
	
	# From here, we'll unlink if it exists. If it doesn't exist, we still exit sucessfully.
	if (-e $file_path)
	{
		# Remove it
		unlink $file_path;
		if (-e $file_path)
		{
			# Failed.
			my $error = $!;
			$anvil->Job->update_progress({
				progress   => 100,
				message    => "error_0180,!!file_path!".$file_path."!!,!!error!".$error."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0180", variables => { 
				file_path => $file_path, 
				error     => $error, 
			}});
			$anvil->nice_exit({exit_code => 1});
		}
		else
		{
			# Success!
			$anvil->Job->update_progress({
				progress => 100,
				message  => "job_0134,!!file_path!".$file_path."!!", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0134", variables => { file_path => $file_path }});
			$anvil->nice_exit({exit_code => 0});
		}
	}
	else
	{
		# Doesn't exist, nothing to do.
		$anvil->Job->update_progress({
			progress => 100,
			message  => "job_0135,!!file_path!".$file_path."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "job_0135", variables => { file_path => $file_path }});
		$anvil->nice_exit({exit_code => 0});
	}
	
	return(0);
}

sub process_incoming_file
{
	my ($anvil) = @_;
	
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress},
		message  => "message_0191", 
	});
	
	my $file = ($anvil->data->{jobs}{job_data} =~ /file=(.*)$/)[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }});
	if (not $file)
	{
		# Can't do anything, file wasn't parsed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0170,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0170", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		
		$anvil->nice_exit({exit_code => 1});
	}
	elsif (not -e $file)
	{
		# Can't do anything, file doesn't exist
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0171,!!file!".$file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Move it over to files.
	$anvil->data->{sys}{progress} = 10;
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress}, 
		message  => "message_0192,!!file!".$file."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0192", variables => { file => $file }});
	
	$anvil->Storage->move_file({
		debug       => 2,
		overwrite   => 1, 
		source_file => $file,
		target_file => $anvil->data->{path}{directories}{shared}{files}."/", 
	});

	
	my $file_name   = ($file =~ /\/.*\/(.*?)$/)[0];
	my $target_file = $anvil->data->{path}{directories}{shared}{files}."/".$file_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		file_name   => $file_name,
		target_file => $target_file, 
	}});
	
	if (not -e $target_file)
	{
		# Failed to move.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0172,!!file!".$file."!!,!!target_directory!".$anvil->data->{path}{directories}{shared}{files}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Change the owner as it'll be apache, which won't be a valid users on anvil members.
	$anvil->Storage->change_owner({
		debug => 2, 
		path  => $target_file, 
		user  => "root", 
		group => "root",
	});
	
	# Calculate the md5sum.
	$anvil->data->{sys}{progress} = 20;
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress}, 
		message  => "message_0193", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0193"});
	
	my ($string, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{md5sum}." ".$target_file});
	my $md5sum = ($string =~ /^(.*?)\s/)[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		string      => $string, 
		md5sum      => $md5sum,
		return_code => $return_code,
	}});
	
	# Store the file details!
	$anvil->data->{sys}{progress} = 30;
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress}, 
		message  => "message_0194,!!md5sum!".$md5sum."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0194", variables => { md5sum => $md5sum }});
	
	$anvil->Storage->get_file_stats({
		debug     => 2,
		file_path => $target_file, 
	});
	
	my $file_mimetype = $anvil->data->{file_stat}{$target_file}{mimetype};
	my $file_size     = $anvil->data->{file_stat}{$target_file}{size};
	my $file_mtime    = $anvil->data->{file_stat}{$target_file}{modified_time}; 
	my $executable    = -x $target_file ? 1 : 0;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		file_mimetype => $file_mimetype, 
		file_size     => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
		file_mtime    => $file_mtime,
		executable    => $executable, 
	}});
	
	# This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 
	# 'other'. If set to 'DELETED', the file will be removed from disk.
	my $file_type = "other";
	if ($file_mimetype =~ /cd-image/)
	{
		$file_type = "iso";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_type => $file_type }});
	}
	# This will need to be expanded over time
	elsif (($executable) or ($file_mimetype =~ /perl/) or ($file_mimetype =~ /python/))
	{
		$file_type = "script";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_type => $file_type }});
		
		# Change the mode to be executable
		$anvil->Storage->change_mode({
			debug => 2, 
			path  => $target_file, 
			mode  => "0755",
		});
	}
	elsif ($file_mimetype =~ /raw-disk-image/)
	{
		$file_type = "image";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_type => $file_type }});
	}
	
	my $file_uuid = $anvil->Database->insert_or_update_files({
		debug          => 2,
		file_name      => $file_name, 
		file_directory => $anvil->data->{path}{directories}{shared}{files}, 
		file_size      => $file_size, 
		file_md5sum    => $md5sum, 
		file_type      => $file_type, 
		file_mtime     => $file_mtime, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
	
	# Now copy this to our peers.
	foreach my $host_uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
	{
		# Periodically, autovivication causes and empty key to appear.
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
		next if ((not $host_uuid) or (not $anvil->Validate->uuid({uuid => $host_uuid})));
		next if $host_uuid eq $anvil->Get->host_uuid;
		
		my $host     = $anvil->data->{database}{$host_uuid}{host};
		my $password = $anvil->data->{database}{$host_uuid}{password};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			host     => $host,
			password => $anvil->Log->is_secure($password), 
		}});
		
		my $striker_name = $anvil->Get->host_name_from_uuid({host_uuid => $host_uuid});
		my $say_host     = $striker_name." (".$host.")";
		
		# Rsync the file.
		$anvil->data->{sys}{progress} += 10;
		$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
		$anvil->Job->update_progress({
			progress => $anvil->data->{sys}{progress}, 
			message  => "message_0195,!!host!".$say_host."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => { host => $say_host }});
		
		$anvil->Storage->rsync({
			debug       => 2, 
			source      => $target_file, 
			destination => "root\@".$host.":".$anvil->data->{path}{directories}{shared}{files}."/", 
			try_again   => 1, 
		});
	}
	
	# Tell other Anvil! systems to download this file.
	$anvil->Database->get_anvils({debug => 2});
	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
	{
		my $anvil_uuid            = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
		my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node1_host_uuid};
		my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node2_host_uuid};
		my $anvil_dr1_host_uuid   = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			anvil_uuid            => $anvil_uuid, 
			anvil_node1_host_uuid => $anvil_node1_host_uuid, 
			anvil_node2_host_uuid => $anvil_node2_host_uuid, 
			anvil_dr1_host_uuid   => $anvil_dr1_host_uuid, 
		}});
		
		$anvil->data->{sys}{progress} += 5;
		$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
		$anvil->Job->update_progress({
			progress => $anvil->data->{sys}{progress}, 
			message  => "message_0196,!!anvil_name!".$anvil_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => { anvil_name => $anvil_name }});
		
		my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({
			debug                    => 2, 
			file_location_file_uuid  => $file_uuid, 
			file_location_anvil_uuid => $anvil_uuid, 
			file_location_active     => 1, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
		
		# Register a job to call anvil-sync-shared 
		foreach my $host_uuid ($anvil_node1_host_uuid, $anvil_node2_host_uuid, $anvil_dr1_host_uuid)
		{
			next if not $host_uuid;
			my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
				file            => $THIS_FILE, 
				line            => __LINE__, 
				job_command     => $anvil->data->{path}{exe}{'anvil-sync-shared'}.$anvil->Log->switches, 
				job_data        => "file_uuid=".$file_uuid, 
				job_name        => "storage::pull_file", 
				job_title       => "job_0132", 
				job_description => "job_0133", 
				job_progress    => 0,
				job_host_uuid   => $host_uuid,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
		}
	}
	
	# Done!
	$anvil->Job->update_progress({
		progress => 100, 
		message  => "message_0197", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0197"});
	
	return(0);
}

# This pulls files from dashboards onto the running host.
sub process_pull_file
{
	my ($anvil) = @_;
	
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress},
		message  => "message_0198", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0198" });
	
	# Collect some data
	$anvil->Database->get_anvils({debug => 3});
	$anvil->Database->get_files({debug => 3});
	$anvil->Database->get_file_locations({debug => 3});
	
	my $file_uuid  = ($anvil->data->{jobs}{job_data} =~ /file_uuid=(.*)$/)[0];
	my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({debug => 2});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_uuid => $anvil_uuid,
		file_uuid  => $file_uuid, 
	}});
	
	if (not $file_uuid)
	{
		# Can't do anything, file wasn't parsed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0173,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0173", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		$anvil->nice_exit({exit_code => 1});
	}
	elsif (not exists $anvil->data->{files}{file_uuid}{$file_uuid})
	{
		# File UUID doesn't appear to be valid.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0174,!!file_uuid!".$file_uuid."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0174", variables => { file_uuid => $file_uuid }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	my $file_name      = $anvil->data->{files}{file_uuid}{$file_uuid}{file_name};
	my $file_directory = $anvil->data->{files}{file_uuid}{$file_uuid}{file_directory};
	my $file_path      = $file_directory."/".$file_name;
	my $file_size      = $anvil->data->{files}{file_uuid}{$file_uuid}{file_size};
	my $file_md5sum    = $anvil->data->{files}{file_uuid}{$file_uuid}{file_md5sum};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		's1:file_name'      => $file_name, 
		's2:file_directory' => $file_directory, 
		's3:file_path'      => $file_path, 
		's4:file_size'      => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
		's5:file_md5sum'    => $file_md5sum, 
	}});
	
	if (not $anvil_uuid)
	{
		# Uhhh...
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0175,!!file!".$file_path."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0175", variables => { file => $file_path }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# How many Strikers are up and have the file we're looking for?
	$anvil->data->{target_strikers} = [];
	foreach my $host_uuid (keys %{$anvil->data->{database}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_uuid => $host_uuid }});
		
		my $host_name = $anvil->Get->host_name_from_uuid({debug => 2, host_uuid => $host_uuid});
		my $target    = $anvil->data->{database}{$host_uuid}{host};
		my $password  = $anvil->data->{database}{$host_uuid}{password};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			host_name => $host_name, 
			target    => $target,
			password  => $anvil->Log->is_secure($password), 
		}});
		
		# If the file exists, the return code is '0'. If the file isn't found, '1' is returned.
		# When found, the size in bytes followed by the file name is returned.
		my $shell_call = $anvil->data->{path}{exe}{wc}." -c ".$file_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		my ($output, $error, $return_code) = $anvil->Remote->call({
			debug       => 3, 
			shell_call  => $shell_call, 
			target      => $target,
			password    => $password,
			remote_user => "root", 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			output => $output, 
			error  => $error,
			output => $output,
		}});
		if ($output =~ /^(\d+)\s+$file_path$/)
		{
			my $size_on_peer = $1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				size_on_peer => $size_on_peer." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $size_on_peer}).")"
			}});
			
			# For now, we only do a size check as md5sums can take a long time.
			if ($size_on_peer eq $file_size)
			{
				# We can pull from this striker!
				push @{$anvil->data->{target_strikers}}, {
					host_name => $host_name,
					host_uuid => $host_uuid,
					target    => $target,
					password  => $password,
				};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					host_name => $host_name,
					host_uuid => $host_uuid,
					target    => $target,
					password  => $anvil->Log->is_secure($password),
				}});
			}
			else
			{
				# The file doesn't exist or we couldn't contact the Striker, so we'll skip 
				# it.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0576", variables => { 
					file_path => $file_path,
					host_name => $host_name, 
				}});
			}
		}
	}
	
	my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
	my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
	my $anvil_dr1_host_uuid   = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		anvil_node1_host_uuid => $anvil_node1_host_uuid, 
		anvil_node2_host_uuid => $anvil_node2_host_uuid, 
		anvil_dr1_host_uuid   => $anvil_dr1_host_uuid, 
	}});
	
	# Where we pull from will depend on which machine we are and how many strikers we have. If we have
	# one Anvil!, node 1 and 2 download at the same time, and DR waits. If there are two strikers, Each
	# node will download from a different striker (if possible) and DR waits. If there are 3 or more 
	# Strikers, DR does not wait, and downloads from a different striker than the striker's nodes use.
	my $i_am = "node1";
	if ($anvil->Get->host_uuid eq $anvil_node2_host_uuid)
	{
		$i_am = "node2";
	}
	elsif ($anvil->Get->host_uuid eq $anvil_dr1_host_uuid)
	{
		$i_am = "dr1";
		
		# As we're DR, we'll likely be pinging the nodes to seee if they're up when we wait for them
		# to finish jobs. As such, load their IPs into memory.
		$anvil->Network->load_ips({debug => 2, host_uuid => $anvil_node1_host_uuid});
		$anvil->Network->load_ips({debug => 2, host_uuid => $anvil_node2_host_uuid});
	}
	
	my $striker_count = @{$anvil->data->{target_strikers}};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		i_am          => $i_am, 
		striker_count => $striker_count, 
	}});
	if (not $striker_count)
	{
		# No available Strikers.
		$anvil->Job->update_progress({
			progress   => 1,
			message    => "warning_0072,!!file_path!".$file_path."!!", 
			job_status => "failed", 
		});
		sleep 60;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "warning_0072", variables => { file_path => $file_path }});
		$anvil->nice_exit({exit_code => 2});
	}
	
	my $use     = 0;
	my $dr_wait = 1;
	if ($striker_count >= 3)
	{
		$dr_wait = 0;
		if ($i_am eq "node1")    { $use = 0; }
		elsif ($i_am eq "node2") { $use = 1; }
		elsif ($i_am eq "dr1")   { $use = 2; }
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'use'   => $use, 
			dr_wait => $dr_wait,
		}});
	}
	elsif ($striker_count == 2)
	{
		# Two strikers, each node will use the other Striker, DR waits for both to be done.
		if ($i_am eq "node1")    { $use = 0; }
		elsif ($i_am eq "node2") { $use = 1; }
		elsif ($i_am eq "dr1")   { $use = 1; }
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'use'   => $use, 
			dr_wait => $dr_wait,
		}});
	}
	elsif ($striker_count == 1)
	{
		# Only 1 Striker
		if ($i_am eq "node1")    { $use = 0; }
		elsif ($i_am eq "node2") { $use = 0; }
		elsif ($i_am eq "dr1")   { $use = 0; }
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			'use'   => $use, 
			dr_wait => $dr_wait,
		}});
	}
	
	# If I'm DR and need to wait, look for jobs on node1 and node2 and wait until both are done (or is 
	# offline).
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		i_am    => $i_am, 
		dr_wait => $dr_wait,
	}});
	if (($i_am eq "dr1") && ($dr_wait))
	{
		my $node1_job_uuid = "";
		my $node1_online   = 1;
		my $node2_job_uuid = "";
		my $node2_online   = 1;
		my $node1_waiting  = 1;
		my $node2_waiting  = 1;
		
		my $query = "
SELECT 
    job_uuid, 
    job_host_uuid 
FROM 
    jobs 
WHERE 
    job_name = 'storage::pull_file' 
AND 
    (
        job_host_uuid = ".$anvil->Database->quote($anvil_node1_host_uuid)." 
    OR 
        job_host_uuid = ".$anvil->Database->quote($anvil_node2_host_uuid)."
    )
;";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
		
		my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
		my $count   = @{$results};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			results => $results, 
			count   => $count, 
		}});
		foreach my $row (@{$results})
		{
			my $job_uuid      = $row->[0];
			my $job_host_uuid = $row->[1];
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				job_uuid      => $job_uuid,
				job_host_uuid => $job_host_uuid, 
			}});
			
			if ($job_host_uuid eq $anvil_node1_host_uuid)
			{
				$node1_job_uuid = $job_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_job_uuid => $node1_job_uuid }});
			}
			if ($job_host_uuid eq $anvil_node2_host_uuid)
			{
				$node2_job_uuid = $job_uuid;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_job_uuid => $node2_job_uuid }});
			}
		}
		
		my $waiting = 1;
		while($waiting)
		{
			if ($node1_waiting)
			{
				if (not $node1_job_uuid)
				{
					$node1_waiting = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_waiting => $node1_waiting }});
				}
				else
				{
					$node1_waiting = wait_on_host($anvil, $anvil_node1_host_uuid, $node1_job_uuid);
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node1_waiting => $node1_waiting }});
				}
			}
			if ($node2_waiting)
			{
				if (not $node2_job_uuid)
				{
					$node2_waiting = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_waiting => $node2_waiting }});
				}
				else
				{
					$node2_waiting = wait_on_host($anvil, $anvil_node2_host_uuid, $node2_job_uuid);
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { node2_waiting => $node2_waiting }});
				}
			}
			
			if ((not $node1_waiting) && (not $node2_waiting))
			{
				# We can proceed.
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
			else
			{
				# Sleep for a bit.
				my $sleep_time = 30;
				my $wait_until = $anvil->Get->date_and_time({offset => $sleep_time});
				
				$anvil->Job->update_progress({
					progress   => 1,
					message    => "message_0199,!!strikers!".$striker_count."!!,!!node1_waiting!".$node1_waiting."!!,!!node2_waiting!".$node2_waiting."!!,!!wait_until!".$wait_until."!!", 
					job_status => "failed", 
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0199", variables => { 
					strikers      => $striker_count,
					node1_waiting => $node1_waiting, 
					node2_waiting => $node2_waiting, 
					wait_until    => $wait_until,
				}});
				
				sleep $sleep_time;
			}
		}
	}
	
	# Now proceed with the download!
	my $target_host_name = $anvil->data->{target_strikers}->[$use]->{host_name};
	my $target_host_uuid = $anvil->data->{target_strikers}->[$use]->{host_uuid};
	my $target_host      = $anvil->data->{target_strikers}->[$use]->{target};
	my $target_password  = $anvil->data->{target_strikers}->[$use]->{password};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		target_host_name => $target_host_name,
		target_host_uuid => $target_host_uuid, 
		target_host      => $target_host, 
		target_password  => $anvil->Log->is_secure($target_password),
	}});
	
	# Rsync the file.
	my $say_source_file  = "root\@".$target_host.":".$file_path;
	my $target_directory = $file_directory."/";
	$anvil->Job->update_progress({
		progress => 50, 
		message  => "message_0200,!!source_file!".$say_source_file."!!,!!target_directory!".$target_directory."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0200", variables => { 
		source_file      => $say_source_file, 
		target_directory => $target_directory, 
	}});
	
	$anvil->Storage->rsync({
		debug       => 2, 
		source      => $say_source_file, 
		destination => $target_directory, 
		try_again   => 1, 
	});
	
	if (-e $file_path)
	{
		# Calculate the md5sum.
		$anvil->Job->update_progress({
			progress => 75, 
			message  => "message_0201", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0201"});
		
		my $local_md5sum = $anvil->Get->md5sum({file => $file_path});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_md5sum => $local_md5sum }});
		
		if ($file_md5sum eq $local_md5sum)
		{
			$anvil->Job->update_progress({
				progress => 100,
				message  => "message_0202", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0202"});
			$anvil->nice_exit({exit_code => 0});
		}
		else
		{
			# Unlink the file. The perioding sync call can try again later.
			unlink $file_path;
			$anvil->Job->update_progress({
				progress   => 1,
				message    => "error_0176,!!local_md5sum!".$local_md5sum."!!,!!file_md5sum!".$file_md5sum."!!", 
				job_status => "failed", 
			});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0176", variables => { 
				local_md5sum => $local_md5sum,
				file_md5sum  => $file_md5sum,
			}});
			sleep 60;
			$anvil->nice_exit({exit_code => 1});
		}
	}
	else
	{
		# Failed... 
		$anvil->Job->update_progress({
			progress   => 1,
			message    => "error_0177", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0177" });
		sleep 60;
		$anvil->nice_exit({exit_code => 1});
	}
	
	return(0);
}

# This takes a host and job UUID and determines if we're still waiting on the target.
sub wait_on_host
{
	my ($anvil, $host_uuid, $job_uuid) = @_;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_uuid => $host_uuid." (".$anvil->Get->host_name_from_uuid({host_uuid => $host_uuid}).")", 
		job_uuid  => $job_uuid,
	}});
	
	my $waiting = 1;
	
	# Look up the job progress.
	my $query = "SELECT job_progress FROM jobs WHERE job_uuid = ".$anvil->Database->quote($job_uuid).";";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
	
	my $progress = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }});
	
	if ($progress == 100)
	{
		$waiting = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
	}
	else
	{
		# Can I ping the node?
		my $pinged = 0;
		foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{$host_uuid}{interface}})
		{
			my $target_ip = $anvil->data->{network}{$host_uuid}{interface}{$interface}{ip};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				interface => $interface, 
				target_ip => $target_ip,
			}});
			
			($pinged, my $average_time) = $anvil->Network->ping({
				ping  => $target_ip, 
				count => 1,
			});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pinged => $pinged }});
			
			last if $pinged;
		}
		
		if (not $pinged)
		{
			# Stop waiting, it looks to be offline.
			$waiting = 0;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
		}
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
	return($waiting)
}