anvil/tools/striker-sync-shared

#!/usr/bin/perl
# 
# This runs on striker dashboards and syncs files under /mnt/shared on all known systems. It reaches out and 
# pulls over any files under /mnt/shared/files/ to the same on the local system. It then pushes files out to
# all members of the same Anvil!. 
# 
# If this is called with a job-uuid, file-specific tasks will be handled, like moving files uploaded over a
# browser or deleting / purging a file.
# 
# NOTE: This file is NOT responsible for sync'ing definition files! That is handles in scan-server.
# 
# TODO:
# - Handle deleting files by user input, or if a given file that was on an Anvil! has been removed for both
#   nodes and DR, where applicable.
# - 
 
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});

# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->Get->switches;

# Connect to the database(s).
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});

# If we don't have a job-uuid, look for one.
if (not $anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({debug => 2, program => $THIS_FILE});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
}

# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'})
{
	# Load the job data.
	$anvil->Job->get_job_details({debug => 2});
	$anvil->Job->clear();
	
	$anvil->data->{sys}{progress} = 1;
	if ($anvil->data->{jobs}{job_name} eq "upload::move_incoming")
	{
		process_incoming_file($anvil);
	}
	
	# Job data will be in $anvil->data->{jobs}{job_data}
}
else
{
	# Do a normal periodic search.
	### NOTE: When finding new files, check the size, sleep for 30 seconds, and check again. If a file's 
	###       size changed, skip it, it's likely still being updated.
}

$anvil->nice_exit({exit_code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

sub process_incoming_file
{
	my ($anvil) = @_;
	
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress},
		message  => "message_0191", 
	});
	
	my $file = ($anvil->data->{jobs}{job_data} =~ /file=(.*)$/)[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }});
	if (not $file)
	{
		# Can't do anything, file wasn't parsed.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0170,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0170", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }});
		
		$anvil->nice_exit({exit_code => 1});
	}
	elsif (not -e $file)
	{
		# Can't do anything, file doesn't exist
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0171,!!file!".$file."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Move it over to files.
	$anvil->data->{sys}{progress} = 10;
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress}, 
		message  => "message_0192,!!file!".$file."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0192", variables => { file => $file }});
	
	$anvil->Storage->move_file({
		debug       => 2,
		overwrite   => 1, 
		source_file => $file,
		target_file => $anvil->data->{path}{directories}{shared}{files}."/", 
	});
	
	my $file_name   = ($file =~ /\/.*\/(.*?)$/)[0];
	my $target_file = $anvil->data->{path}{directories}{shared}{files}."/".$file_name;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		file_name   => $file_name,
		target_file => $target_file, 
	}});
	
	if (not -e $target_file)
	{
		# Failed to move.
		$anvil->Job->update_progress({
			progress   => 100,
			message    => "error_0172,!!file!".$file."!!,!!target_directory!".$anvil->data->{path}{directories}{shared}{files}."!!", 
			job_status => "failed", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Calculate the md5sum.
	$anvil->data->{sys}{progress} = 20;
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress}, 
		message  => "message_0193", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0193"});
	
	my ($string, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{md5sum}." ".$target_file});
	my $md5sum = ($string =~ /^(.*?)\s/)[0];
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		string      => $string, 
		md5sum      => $md5sum,
		return_code => $return_code,
	}});
	
	# Store the file details!
	$anvil->data->{sys}{progress} = 30;
	$anvil->Job->update_progress({
		progress => $anvil->data->{sys}{progress}, 
		message  => "message_0194,!!md5sum!".$md5sum."!!", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0194", variables => { md5sum => $md5sum }});
	
	$anvil->Storage->get_file_stats({
		debug     => 2,
		file_path => $target_file, 
	});
	
	my $file_mimetype = $anvil->data->{file_stat}{$target_file}{mimetype};
	my $file_size     = $anvil->data->{file_stat}{$target_file}{size};
	my $file_mtime    = $anvil->data->{file_stat}{$target_file}{modified_time}; 
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		file_mimetype => $file_mimetype, 
		file_size     => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")",
		file_mtime    => $file_mtime,
	}});
	
	# This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 
	# 'other'. If set to 'DELETED', the file will be removed from disk.
	my $file_type = "other";
	if ($file_mimetype =~ /cd-image/)
	{
		$file_type = "iso";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_type => $file_type }});
	}
	
	my $file_uuid = $anvil->Database->insert_or_update_files({
		debug          => 2,
		file_name      => $file_name, 
		file_directory => $anvil->data->{path}{directories}{shared}{files}, 
		file_size      => $file_size, 
		file_md5sum    => $md5sum, 
		file_type      => $file_type, 
		file_mtime     => $file_mtime, 
	});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
	
	# Now copy this to our peers.
	foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
	{
		# Periodically, autovivication causes and empty key to appear.
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid => $uuid }});
		next if ((not $uuid) or (not $anvil->Validate->uuid({uuid => $uuid})));
		next if $uuid eq $anvil->Get->host_uuid;
		
		my $host     = $anvil->data->{database}{$uuid}{host};
		my $password = $anvil->data->{database}{$uuid}{password};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			host     => $host,
			password => $anvil->Log->is_secure($password), 
		}});
		
		my $striker_name = $anvil->Get->host_name_from_uuid({host_uuid => $uuid});
		my $say_host     = $striker_name." (".$host.")";
		
		# Rsync the file.
		$anvil->data->{sys}{progress} += 10;
		$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
		$anvil->Job->update_progress({
			progress => $anvil->data->{sys}{progress}, 
			message  => "message_0195,!!host!".$say_host."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => { host => $say_host }});
		
		$anvil->Storage->rsync({
			debug       => 2, 
			source      => $target_file, 
			destination => "root\@".$host.":".$anvil->data->{path}{directories}{shared}{files}."/", 
			try_again   => 1, 
		});
	}
	
	### TODO: Make is an upload-time option to choose if the uploaded file automatically goes to any given Anvil!
	# Tell other Anvil! systems to download this file.
	$anvil->Database->get_anvils({debug => 2});
	foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
	{
		my $anvil_uuid            = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
		my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node1_host_uuid};
		my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node2_host_uuid};
		my $anvil_dr1_host_uuid   = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			anvil_uuid            => $anvil_uuid, 
			anvil_node1_host_uuid => $anvil_node1_host_uuid, 
			anvil_node2_host_uuid => $anvil_node2_host_uuid, 
			anvil_dr1_host_uuid   => $anvil_dr1_host_uuid, 
		}});
		
		$anvil->data->{sys}{progress} += 5;
		$anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90;
		$anvil->Job->update_progress({
			progress => $anvil->data->{sys}{progress}, 
			message  => "message_0196,!!anvil_name!".$anvil_name."!!", 
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => { anvil_name => $anvil_name }});
		
		my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({
			debug                    => 2, 
			file_location_file_uuid  => $file_uuid, 
			file_location_anvil_uuid => $anvil_uuid, 
			file_location_active     => 1, 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
		
		### TODO: Register a job for each member to run this file with an 'upload::pull_file' with 
		###       the job_data being the 'file=$target_file'. Have node1 try the first striker, and 
		###       node2 from the second striker, if it exists. DR should watch node1 and node2's
		###       jobs and not download until those jobs hit 100.
	}
	
	# Done!
	$anvil->Job->update_progress({
		progress => 100, 
		message  => "message_0197", 
	});
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0197"});
	
	return(0);
}