anvil/ocf/alteeve/server

#!/usr/bin/perl
# 
#   This is the resource agent used to manage servers on the Anvil! Intelligent Availability platform.
#
#   License: GNU General Public License (GPL) v2+
#            (c) 1997-2019 - Alteeve's Niche! Inc.
# 
# WARNING: This is a pretty purpose-specific resource agent. No effort was made to test this on an rgmanager
#          cluster or on any configuration outside how the Anvil! m3 uses it. If you plan to adapt it to 
#          another purpose, let us know and we'll try to help.
# 
# Based on: https://github.com/ClusterLabs/resource-agents/blob/master/doc/dev-guides/ra-dev-guide.asc
# 
# Error types from pacemaker's perspective;
# 
# - Soft Error -  Unless specifically configured otherwise, pacemaker will attempt to recover a resource 
#                 in-place - usually by restarting the resource on the same node.
# - Hard Error -  Unless specifically configured otherwise, pacemaker will attempt to recover a resource 
#                 which failed with this error by restarting the resource on a different node.
# - Fatal Error - This is a cluster-wide error, it would make no sense to recover such a resource on a 
#                 different node, let alone in-place. When a resource fails with this error, Pacemaker will 
#                 attempt to shut down the resource, and wait for administrator intervention.
# 
# Exit codes;
# 0 - OCF_SUCCESS 
#   - The action completed successfully. This is the expected return code for any successful start, stop, 
#     migrate_to, meta_data, help, and usage action.
#   - For monitor, however, a modified convention applies:
#     - If the server is running we return, OCF_SUCCESS.  If not running and gracefully stopped or migrated 
#       off, return OCF_NOT_RUNNING.
#       
# 1 - OCF_ERR_GENERIC
#   - The action returned a generic error. This is used only when none of the more specific error codes, 
#     defined below, accurately describes the problem.
#   - Pacemaker interprets this exit code as a soft error. 
#     
# 2 - OCF_ERR_ARGS
#   - The resource’s configuration is not valid on this machine. This can happen if the serve fails to boot
#     because of a missing bridge, for example.
#    
# 3 - OCF_ERR_UNIMPLEMENTED
#   - The resource agent was instructed to execute an action that we do not implement.
#   - Not all resource agent actions are mandatory. We don't implement 'promote' or 'demote'. We do implement
#     'migrate_to', 'migrate_from', and 'notify'. If we're misconfigured as a master/slave resource, for 
#     example, then will alert the user about this misconfiguration by returning OCF_ERR_UNIMPLEMENTED.
#     
# 4 - OCF_ERR_PERM 
#   - The action failed due to insufficient permissions. This may be due to a node not being able to open a
#     definition file or resource config. 
#   - Pacemaker interprets this exit code as a hard error. 
#     
# 5 - OCF_ERR_INSTALLED
#   - The action failed because a required component is missing on the node where the action was executed. 
#     This may be due to a required binary not being executable, or a the DRBD resource config file not 
#     existing.
#   - Pacemaker interprets this exit code as a hard error.
#     
# 6 - OCF_ERR_CONFIGURED
#   - The action failed because the user misconfigured the resource in pacemaker. For example, the user may 
#     have configured an alphanumeric string for a parameter that really should be an integer.
#   - Pacemaker interprets this exit code as a fatal error.
#     
# 7 - OCF_NOT_RUNNING
#   - The resource was found not to be running. This is an exit code that may be returned by the monitor 
#     action exclusively. Note that this implies that the resource has either gracefully shut down, or has 
#     never been started.
#     
# 8 - OCF_RUNNING_MASTER
# 9 - OCF_FAILED_MASTER
#   - These OCF exit codes are not used here.
#     

# NOTE: We don't use Anvil::Tools to keep overhead low and to keep this agent independent as possible.
use strict;
use warnings;
use Anvil::Tools;
use XML::Simple;
use JSON;
use Math::BigInt;
use Data::Dumper;

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

# NOTE: Setting 'log_level' and 'log_secure' here will get overridden in the main lopp. Use the Log methods 
#       in the loop as well to override defaults in code.
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});

### Read or Set the environment variables
# This is the name of the server we're managing.                # Example values:
$anvil->data->{environment}{OCF_RESKEY_name}                    = defined $ENV{OCF_RESKEY_name}                    ? $ENV{OCF_RESKEY_name}                    : ""; # srv01-c7 
# This is our node name
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node}        = defined $ENV{OCF_RESKEY_CRM_meta_on_node}        ? $ENV{OCF_RESKEY_CRM_meta_on_node}        : ""; # el8-a01n01.digimer.ca
# This says "UUID", but it's the node ID.
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node_uuid}   = defined $ENV{OCF_RESKEY_CRM_meta_on_node_uuid}   ? $ENV{OCF_RESKEY_CRM_meta_on_node_uuid}   : ""; # 1
# This is the timeout for the called action in millisecond.
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_timeout}        = defined $ENV{OCF_RESKEY_CRM_meta_timeout}        ? $ENV{OCF_RESKEY_CRM_meta_timeout}        : ""; # 20000
# If this is set, we'll bump our log level as well.
$anvil->data->{environment}{PCMK_debug}                         = defined $ENV{PCMK_debug}                         ? $ENV{PCMK_debug}                         : ""; # 0
# These are other variables that are set, but we don't currently care about them
$anvil->data->{environment}{OCF_EXIT_REASON_PREFIX}             = defined $ENV{OCF_EXIT_REASON_PREFIX}             ? $ENV{OCF_EXIT_REASON_PREFIX}             : ""; # ocf-exit-reason:
$anvil->data->{environment}{OCF_RA_VERSION_MAJOR}               = defined $ENV{OCF_RA_VERSION_MAJOR}               ? $ENV{OCF_RA_VERSION_MAJOR}               : ""; # 1
$anvil->data->{environment}{OCF_RA_VERSION_MINOR}               = defined $ENV{OCF_RA_VERSION_MINOR}               ? $ENV{OCF_RA_VERSION_MINOR}               : ""; # 0
$anvil->data->{environment}{OCF_RESKEY_crm_feature_set}         = defined $ENV{OCF_RESKEY_crm_feature_set}         ? $ENV{OCF_RESKEY_crm_feature_set}         : ""; # 3.0.12
$anvil->data->{environment}{OCF_RESOURCE_INSTANCE}              = defined $ENV{OCF_RESOURCE_INSTANCE}              ? $ENV{OCF_RESOURCE_INSTANCE}              : ""; # srv01-c7
$anvil->data->{environment}{OCF_RESOURCE_PROVIDER}              = defined $ENV{OCF_RESOURCE_PROVIDER}              ? $ENV{OCF_RESOURCE_PROVIDER}              : ""; # alteeve
$anvil->data->{environment}{OCF_RESOURCE_TYPE}                  = defined $ENV{OCF_RESOURCE_TYPE}                  ? $ENV{OCF_RESOURCE_TYPE}                  : ""; # server
$anvil->data->{environment}{OCF_ROOT}                           = defined $ENV{OCF_ROOT}                           ? $ENV{OCF_ROOT}                           : ""; # /usr/lib/ocf
# These are set during a migration
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source} = defined $ENV{OCF_RESKEY_CRM_meta_migrate_source} ? $ENV{OCF_RESKEY_CRM_meta_migrate_source} : ""; # el8-a01n01.digimer.ca
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = defined $ENV{OCF_RESKEY_CRM_meta_migrate_target} ? $ENV{OCF_RESKEY_CRM_meta_migrate_target} : ""; # el8-a01n02.digimer.ca
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_record_pending} = defined $ENV{OCF_RESKEY_CRM_meta_record_pending} ? $ENV{OCF_RESKEY_CRM_meta_record_pending} : ""; # true

# If pacemaker is in debug, so are we,
if ($anvil->data->{environment}{PCMK_debug})
{
	$anvil->Log->level({set => 3});
}

# Originally, this was designed to start and stop a server's DRBD resources on demand. Early testing appears
# to show this prone to higher risk of fencing if something goes wrong. As such, we're changing the default
# behaviour to leave DRBD resources up. Set this to '1' (here or by switch) to revert back to the old 
# behaviour.
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_stop_drbd_resources} = 0;

# Get any command line switches.
$anvil->Get->switches;

if ($anvil->data->{switches}{stop_drbd_resources})
{
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source} = 1;
}

# Something for the logs
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 3, key => "log_0298"});

### TEST: to be removed later
if ($anvil->data->{switches}{test1})
{
	$anvil->data->{environment}{OCF_RESKEY_name}                    = "test_server";
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_timeout}        = 20000;
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node}        = "el8-a01n01.digimer.ca";
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source} = "el8-a01n01.digimer.ca";
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = "el8-a01n02.digimer.ca";
	#print "Running test 1; Migrate: [".$anvil->data->{environment}{OCF_RESKEY_name}."] from: [".$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source}."] to: [".$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target}."]\n";
}
if ($anvil->data->{switches}{test2})
{
	$anvil->data->{environment}{OCF_RESKEY_name}                    = "test_server";
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_timeout}        = 20000;
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node}        = "el8-a01n02.digimer.ca";
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source} = "el8-a01n02.digimer.ca";
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = "el8-a01n01.digimer.ca";
	#print "Running test 2; Migrate: [".$anvil->data->{environment}{OCF_RESKEY_name}."] from: [".$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source}."] to: [".$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target}."]\n";
}
if ($anvil->data->{switches}{test3})
{
	$anvil->data->{environment}{OCF_RESKEY_name}             = "test_server";
	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node} = "el8-a01n01.digimer.ca";
	#print "Running test 3; Boot or shutdown of: [".$anvil->data->{environment}{OCF_RESKEY_name}."].\n";
}

# This is for debugging.
if (($anvil->data->{switches}{monitor})     or 
    ($anvil->data->{switches}{status})      or 
    ($anvil->data->{switches}{'meta-data'}) or 
    ($anvil->data->{switches}{metadaata}))
{
	show_environment($anvil, 3);
}
else
{
	show_environment($anvil, 3);
}

### What are we being asked to do?
# start        - Starts the resource.
# stop         - Shuts down the resource.
# monitor      - (status aliases here) Queries the resource for its state.
# meta-data    - Dumps the resource agent metadata.
# promote      - Turns a resource into the Master role (Master/Slave resources only).
# demote       - Turns a resource into the Slave role (Master/Slave resources only).
# migrate_to   - migration target
# migrate_from - Implement live migration of resources.
# validate-all - Validates a resource’s configuration.
# help         - (usage maps here) Displays a usage message when the resource agent is invoked from the command line, rather than by the cluster manager.
# notify       - Inform resource about changes in state of other clones.

if ($anvil->data->{switches}{start})
{
	# Start the server
	start_server($anvil);
}
elsif ($anvil->data->{switches}{stop})
{
	# Stop the server
	stop_server($anvil);
}
elsif (($anvil->data->{switches}{monitor}) or ($anvil->data->{switches}{status}))
{
	# Report the status of the server.
	server_status($anvil);
}
elsif (($anvil->data->{switches}{metadaata}) or ($anvil->data->{switches}{'meta-data'}))
{
	show_metadata($anvil);
}
elsif ($anvil->data->{switches}{promote})
{
	# We don't support this, so we return OCF_ERR_UNIMPLEMENTED (3)
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0299", variables => { server => $anvil->data->{environment}{OCF_RESKEY_name} }});
	$anvil->nice_exit({exit_code => 3});
}
elsif ($anvil->data->{switches}{demote})
{
	# We don't support this, so we return OCF_ERR_UNIMPLEMENTED (3)
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0300", variables => { server => $anvil->data->{environment}{OCF_RESKEY_name} }});
	$anvil->nice_exit({exit_code => 3});
}
elsif (($anvil->data->{switches}{migrate_to}) or ($anvil->data->{switches}{migrate_from}))
{
	# We don't support this, so we return OCF_ERR_UNIMPLEMENTED (3)
	migrate_server($anvil);
}
elsif ($anvil->data->{switches}{'validate-all'})
{
	# Validate our local config and setup.
	validate_all($anvil);
	$anvil->nice_exit({exit_code => 0});
}
elsif (($anvil->data->{switches}{help}) or ($anvil->data->{switches}{usage}))
{
	# Show the usage information
	show_usage($anvil);
	$anvil->nice_exit({exit_code => 0});
}
elsif ($anvil->data->{switches}{notify})
{
	# We don't implement this
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level =>0, key => "log_0301"});
	$anvil->nice_exit({exit_code => 3});
}
else
{
	# We were called in some unexpected way. Log an error, show usage and exit.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level =>0, key => "log_0302"});
	show_environment($anvil, 0);
	$anvil->nice_exit({exit_code => 1});
}

# If we hit here, something very wrong happened.
$anvil->nice_exit({exit_code => 255});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

=cut

STATES

The State field lists what state each domain is currently in. A domain can be in one of the following 
possible states:

running     - The domain is currently running on a CPU
idle        - The domain is idle, and not running or runnable.  This can be caused because the domain is 
              waiting on IO (a traditional wait state) or has gone to sleep because there was nothing else 
              for it to do.
paused      - The domain has been paused, usually occurring through the administrator running virsh suspend.
              When in a paused state the domain will still consume allocated resources like memory, but will
              not be eligible for scheduling by the hypervisor.
in shutdown - The domain is in the process of shutting down, i.e. the guest operating system has been 
              notified and should be in the process of stopping its operations gracefully.
shut off    - The domain is not running.  Usually this indicates the domain has been shut down completely, or
              has not been started.
crashed     - The domain has crashed, which is always a violent ending.  Usually this state can only occur if
              the domain has been configured not to restart on crash.
pmsuspended - The domain has been suspended by guest power management, e.g. entered into s3 state.
=cut

# This boots the server if possible.
sub start_server
{
	my ($anvil) = @_;
	
	# Start procedure;
	# 1. Read the XML definition file and find the backing storage and bridges. Soft error if read fails.
	# 2. Make sure the name matches.
	# 3. Make sure we have enough free RAM.
	# 4. Make sure the emulator exists (can be an issue after migrating from an different gen Anvil!).
	# 5.1. Make sure optical drives with mounted data have the disk present. Soft error if not.
	# 5.2. Find any backing DRBD devices
	# 6. For each DRBD device;
	# 6.1. Make sure the backing LV is ACTIVE. Soft error if not.
	# 6.2. Check if the drbd resource is up. If not, up it.
	# 6.3. Make sure the backing disk is UpToDate. Soft error if not.
	# 6.4. Make sure the backing device is 'Connected' or 'Connecting'. Call a connect if not.
	# 7. Make sure all bridges exist and soft error if not.
	# 8. Start the server.
	
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0303", variables => { server => $server }});
	
	# Make sure things are sane.
	validate_all($anvil);
	
	# Is the server already running somewhere?
	find_server($anvil);
	
	# Start the resource, if needed.
	start_drbd_resource($anvil);
	
	# Still alive? Boot!
	my ($success) = $anvil->Server->boot({debug => 2, server => $server});
	if ($success)
	{
		# Success!
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0309", variables => { server => $server }});
		$anvil->nice_exit({exit_code => 0});
	}
	else
	{
		# WTF?
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0310", variables => { 
			server  => $server, 
			'state' => defined $anvil->data->{server}{location}{$server}{host} ? $anvil->data->{server}{location}{$server}{host} : "#!string!unit_0003!#",
		}});
		$anvil->nice_exit({exit_code => 6});
	}
	
	# If we're still alive, then we didn't see the server in the list of running servers, which is really weird.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0311", variables => { server => $server }});
	$anvil->nice_exit({exit_code => 1});
}

# This stops the DRBD resource(s) that ran under a server.
sub stop_drbd_resource
{
	my ($anvil) = @_;
	
	my $server  = $anvil->data->{environment}{OCF_RESKEY_name};
	my $host    = $anvil->_short_host_name;
	my $peer    = $anvil->data->{drbd}{config}{$host}{peer};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server  => $server,
		host    => $host, 
		peer    => $peer,
	}});
	
	# Start DRBD locally.
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{resource}})
	{
		my $peer_ip = $anvil->data->{drbd}{config}{$host}{resource}{$resource}{connection}{$peer}{ip_address};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0408", variables => { 
			server   => $server,
			peer     => $peer, 
			peer_ip  => $peer_ip, 
			resource => $resource,
		}});
	
		# Bring the peer's resource down.
		$anvil->DRBD->manage_resource({
			resource => $resource, 
			task     => "down",
			target   => $peer_ip, 
		});
		
		# Bring the local resource down
		$anvil->DRBD->manage_resource({
			resource => $resource, 
			task     => "down",
		});
	}
	
	return(0);
}

# This starts the drbd resource(s) for the requested server, if needed.
sub start_drbd_resource
{
	my ($anvil) = @_;
	
	my $server  = $anvil->data->{environment}{OCF_RESKEY_name};
	my $host    = $anvil->_short_host_name;
	my $peer    = $anvil->data->{drbd}{config}{$host}{peer};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server  => $server,
		host    => $host, 
		peer    => $peer,
	}});
	
	# Do we need startup?
	my $startup_needed = 0;
	$anvil->DRBD->get_status({debug => 3});
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{resource}})
	{
		# Is the current resource up locally already? If it is, we're done.
		my $role = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{role};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { role => $role }});
		
		if ((lc($role) ne "secondary") && (lc($role) ne "primary"))
		{
			$startup_needed = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
			last;
		}
		else
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0434", variables => { 
				resource => $resource,
				role     => $role, 
			}});
		}
	}
	
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { startup_needed => $startup_needed }});
	if (not $startup_needed)
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0431"});
		return(0);
	}
	
	# Start DRBD locally.
	foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{resource}})
	{
		my $peer_ip = $anvil->data->{drbd}{config}{$host}{resource}{$resource}{connection}{$peer}{ip_address};
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0419", variables => { 
			server   => $server,
			peer     => $peer, 
			peer_ip  => $peer_ip, 
			resource => $resource,
		}});
		
		# Bring the local resource up
		$anvil->DRBD->manage_resource({
			resource => $resource, 
			task     => "up",
		});
		
		# Bring the peer's resource up.
		$anvil->DRBD->manage_resource({
			resource => $resource, 
			task     => "up",
			target   => $peer_ip, 
		});
		
		# Now wait for it to be connected or UpToDate...
		my $waiting = 1;
		while($waiting)
		{
			$anvil->DRBD->get_status({debug => 3});
			
			my $connection_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer}{'connection-state'};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				connection_state => $connection_state,
			}});
			
			my $all_ready = 1;
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}})
			{
				my $disk_state        = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{devices}{volume}{$volume}{'disk-state'};
				my $replication_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer}{volume}{$volume}{'replication-state'};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					disk_state        => $disk_state,
					replication_state => $replication_state, 
				}});
				
				# Is the peer isn't connected (directly or by being in Sync), or this volume 
				# isn't UpToDate, we need to keep waiting.
				if ((lc($disk_state) ne "uptodate") && ($replication_state !~ /^Sync/i) && (lc($connection_state) ne "connected"))
				{
					$all_ready = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
				}
			}
			
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_ready => $all_ready }});
			if ($all_ready)
			{
				$waiting = 0;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
			}
			
			if ($waiting)
			{
				sleep 1;
			}
		}
	}
	
	# If auto-promote isn't set, promote the resource.
	if (not $anvil->data->{drbd}{config}{$host}{'auto-promote'})
	{
		foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{resource}})
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0420", variables => { 
				server   => $server,
				resource => $resource,
			}});
			# Make the local resource primary.
			$anvil->DRBD->manage_resource({
				resource => $resource, 
				task     => "primary",
			});
		}
	}
	
	return(0);
}

# This uses the DRBD information to find other peers and see if the server is running on them.
sub find_server
{
	my ($anvil) = @_;
	
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	my $host   = $anvil->_short_host_name;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0415", variables => { server => $server }});
	foreach my $ip_address (sort {$a cmp $b} keys %{$anvil->data->{drbd}{config}{$host}{ip_addresses}})
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ip_address => $ip_address }});
		$anvil->Server->find({
			debug       => 3, 
			target      => $ip_address,
			remote_user => "root", 
		});
	}
	
	foreach my $this_server (sort {$a cmp $b} keys %{$anvil->data->{server}{location}})
	{
		my $status = $anvil->data->{server}{location}{$this_server}{status};
		my $host   = $anvil->data->{server}{location}{$this_server}{host};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			this_server => $this_server, 
			status      => $status, 
			host        => $host, 
		}});
	}
	if ((exists $anvil->data->{server}{location}{$server}) && ($anvil->data->{server}{location}{$server}{host}))
	{
		# The server is running. If it is running here, exit with success. If it's running elsewhere,
		# exit with a failure.
		my $status = $anvil->data->{server}{location}{$server}{status};
		my $host   = $anvil->data->{server}{location}{$server}{host};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			status => $status, 
			host   => $host, 
		}});
		
		if ($host eq $anvil->_host_name)
		{
			# Already running, we're good, and we're done.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0416", variables => { server => $server }});
			$anvil->nice_exit({exit_code => 0});
		}
		elsif ($host =~ /dr(\d+)$/)
		{
			# The server is running elsewhere. If the peer host is DR, exit with 
			# OCF_ERR_CONFIGURED (6) so that pacemaker doesn't try to also start the server on 
			# the other node, because we don't know the state of it here.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0417", variables => { 
				server => $server,
				host   => $host, 
			}});
			$anvil->nice_exit({exit_code => 6});
		}
		else
		{
			# It looks like it's running on the peer. So we'll exit OCF_ERR_INSTALLED (5) to tell
			# pacemaker to try to start it on our peer.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0418", variables => { 
				server => $server,
				host   => $host, 
			}});
			$anvil->nice_exit({exit_code => 6});
		}
	}
	
	return(0);
}

# This shuts down the server if possible.
sub stop_server
{
	my ($anvil) = @_;
	
	# Stopping the server is simply a question of "is the server running?" and, if so, stop it. Once 
	# stopped, we stop the DRBD resource on both nodes.
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	
	# Read in an parse the server's XML.
	$anvil->System->check_storage({debug => 2});
	$anvil->Server->get_status({debug => 2, server => $server});
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0313", variables => { server => $server }});
	my $success = $anvil->Server->shutdown({debug => 2, server => $server});
	if (not $success)
	{
		# Something went wrong. Details should be in the logs.
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Now stop the DRBD resource(s).
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		'environment::OCF_RESKEY_CRM_meta_stop_drbd_resources' => $anvil->data->{environment}{OCF_RESKEY_CRM_meta_stop_drbd_resources}, 
	}});
	if ($anvil->data->{environment}{OCF_RESKEY_CRM_meta_stop_drbd_resources})
	{
		stop_drbd_resource($anvil);
	}
	
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0324", variables => { server => $server }});
	$anvil->nice_exit({exit_code => 0});
}

# This checks the status of the server.
sub server_status
{
	my ($anvil) = @_;
	
	# If the named server is running, return OCF_SUCCESS (0), otherwise OCF_NOT_RUNNING (7). If the 
	# server is failed, return OCF_ERR_GENERIC (1).
	my $state  = "";
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	
	### NOTE: When pacemaker is first starting, virsh won't be up right away. So if we get a return code 
	###       of '1', we'll try again up to 50% of 'environment::OCF_RESKEY_CRM_meta_timeout'.
	if (not $anvil->data->{environment}{OCF_RESKEY_CRM_meta_timeout})
	{
		# Set a sane default of 20 seconds.
		$anvil->data->{environment}{OCF_RESKEY_CRM_meta_timeout} = 20000;
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0331", variables => { timeout => $anvil->data->{environment}{OCF_RESKEY_CRM_meta_timeout} }});
	}
	
	$anvil->nice_exit({exit_code => 0});
}

# Migrate the server
sub migrate_server
{
	my ($anvil) = @_;
	
	### NOTE: For now, we're not going to block if the target is not UpToDate. There are times when a
	###       user might want to do this (ie: sync will be done soon and the need to evacuate the node
	###       ASAP is high). Maybe we'll enforce this and require a '--force' switch later?
	# If we were given 'migrate_to', we need to make sure the storage is UpToDate on the peer for all 
	# backing resources. We can't check the target's bridges, but the migation will fail if one is 
	# missing. 
	# If we're given 'migrate_from', we're pulling the server towards us, so we can check both brdiges
	# and storage.
	my $server       = $anvil->data->{environment}{OCF_RESKEY_name};
	my $source       = $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_source};
	my $target       = $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target};
	my $meta_on_node = $anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		server                   => $server, 
		source                   => $source, 
		target                   => $target,
		meta_on_node             => $meta_on_node,
		'switches::migrate_to'   => $anvil->data->{switches}{migrate_to}, 
		'switches::migrate_from' => $anvil->data->{switches}{migrate_from}, 
	}});
	
	# The actual migration command will involve enabling dual primary, then beginning the migration. The
	# virsh call will depend on if we're pushing or pulling. Once the migration completes, regardless of
	# success or failure, dual primary will be disabled again.
	my $migration_command = "";
	my $migrated          = 0;
	if ($anvil->data->{switches}{migrate_to})
	{
		# Can I even connect to the target?
		my ($access) = $anvil->Remote->test_access({debug => 2, target => $target});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { access => $access }});
		if (not $access)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0429", variables => { 
				server => $server,
				target => $target,
			}});
			### TODO: I wonder if this should be exit'ed with '6'?
			$anvil->nice_exit({exit_code => 5});
		}
		
		# Find the server
		$anvil->Server->find({debug => 3});
		my $server_host   = defined $anvil->data->{server}{location}{$server}{host}   ? $anvil->data->{server}{location}{$server}{host}   : "";
		my $server_status = defined $anvil->data->{server}{location}{$server}{status} ? $anvil->data->{server}{location}{$server}{status} : "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_host   => $server_host, 
			server_status => $server_status, 
		}});
		
		# Is it already on the target?
		if (not $server_status)
		{
			# Maybe...
			$anvil->Server->find({debug => 3, target => $target});
			$server_host   = defined $anvil->data->{server}{location}{$server}{host}   ? $anvil->data->{server}{location}{$server}{host}   : "";
			$server_status = defined $anvil->data->{server}{location}{$server}{status} ? $anvil->data->{server}{location}{$server}{status} : "";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				server_host   => $server_host, 
				server_status => $server_status, 
			}});
			
			if (($server_host eq $target) && (($server_status) && ($server_status eq "running")))
			{
				# Already over there, we're done.
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0430", variables => { 
					server => $server,
					target => $target,
				}});
				$anvil->nice_exit({exit_code => 0});
			}
		}
		
		if (not $server_host)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0344", variables => { server => $server }});
			$anvil->nice_exit({exit_code => 1});
		}
		
		# Get a view of the servers locally and our peer.
		validate_all($anvil);
		
		# Get the DRBD status.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0387"});
		$anvil->DRBD->get_status({debug => 2});
		
		# Make sure all resource(s) are ready for the server.
		my $all_up_to_date = 1;
		my $host           = $anvil->_short_host_name;
		my $peer_name      = $anvil->data->{drbd}{config}{$host}{peer};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			host      => $host, 
			peer_name => $peer_name, 
		}});
		foreach my $resource (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{resource}})
		{
			my $connection_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'connection-state'};
			my $peer_node_id     = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'peer-node-id'};
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				peer_node_id     => $peer_node_id, 
				resource         => $resource,
				connection_state => $connection_state, 
			}});
			if (lc($connection_state) ne "connected")
			{
				# Try to bring the resource up on the peer now.
				$anvil->DRBD->manage_resource({
					resource => $resource, 
					task     => "up",
					target   => $target, 
				});
				
				# We'll give it 20 seconds.
				my $wait = 20;
				while($wait)
				{
					$anvil->DRBD->get_status({debug => 2});
					$connection_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{'connection-state'};
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
						resource         => $resource,
						connection_state => $connection_state, 
					}});
					
					if (lc($connection_state) ne "connected")
					{
						# It's up!
						$wait = 0;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'wait' => $wait }});
					}
					else
					{
						$wait--;
						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'wait' => $wait }});
						
						if (not $wait)
						{
							# We're done waiting.
							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0428", variables => { 
								server           => $server,
								target           => $target,
								resource         => $resource,
								connection_state => $connection_state,
							}});
							### TODO: I wonder if this should be exit'ed with '6'?
							$anvil->nice_exit({exit_code => 5});
						}
					}
				}
				
			}
			foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{volume}})
			{
				my $peer_disk_state   = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{volume}{$volume}{'peer-disk-state'};
				my $percent_in_sync   = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{volume}{$volume}{'percent-in-sync'};
				my $replication_state = $anvil->data->{drbd}{status}{$host}{resource}{$resource}{connection}{$peer_name}{volume}{$volume}{'replication-state'};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
					volume            => $volume, 
					peer_disk_state   => $peer_disk_state,
					percent_in_sync   => $percent_in_sync,
					replication_state => $replication_state,
				}});
				
				if (lc($peer_disk_state) ne "uptodate")
				{
					$all_up_to_date = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_up_to_date => $all_up_to_date }});
				}
			}
		}
		
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { all_up_to_date => $all_up_to_date }});
		if (not $all_up_to_date)
		{
			### TODO: If we decide later to block migration to Inconsistent peers, here's where we'd do it.
		}
		
		# If we're still alive, we're ready to migrate.
		($migrated) = $anvil->Server->migrate({
			debug  => 2, 
			server => $server, 
			target => $target 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { migrated => $migrated }});
	}
	elsif ($anvil->data->{switches}{migrate_from})
	{
		# Pull the server here. Start by verifying it's on the 'meta_on_node' host.
		# Scan locally and on our peer
		$anvil->Server->find({debug => 2});
		$anvil->Server->find({debug => 2, target => $meta_on_node, refresh => 0});
		
		my $host       = defined $anvil->data->{server}{location}{$server}{host}   ? $anvil->data->{server}{location}{$server}{host}   : "";
		my $short_host = ($host =~ /^(.*?)\..*$/)[0];
		my $status      = defined $anvil->data->{server}{location}{$server}{status} ? $anvil->data->{server}{location}{$server}{status} : "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			host       => $host, 
			short_host => $short_host, 
			status     => $status, 
			target     => $target,
		}});
		
		# Convert the host to a short name, in case the node's name is the short version.
		
		my $server_host   = defined $anvil->data->{server}{location}{$server}{host}   ? $anvil->data->{server}{location}{$server}{host}   : "";
		my $server_status = defined $anvil->data->{server}{location}{$server}{status} ? $anvil->data->{server}{location}{$server}{status} : "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
			server_host   => $server_host, 
			server_status => $server_status, 
		}});
		
		# This is called after a migration. If that is the case here, the target will be us. Just 
		# make sure it is running and, if so, return '0'. The 'meta_on_node' is the new host.
		if (($target eq $anvil->_host_name) or ($target eq $anvil->_short_host_name) or ($target eq $meta_on_node))
		{
			# If it's running, we're succesfully out.
			if ((($host eq $target) or ($short_host eq $target)) && ($status eq "running"))
			{
				# Success!
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0347", variables => { server => $server }});
				$anvil->nice_exit({exit_code => 0});
			}
			
			# If we're still alive, we'll proceed as if we're pulling the server to us, and maybe
			# that will work.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0348", variables => { server => $server }});
		}
		
		# Validate as if we were about to boot the server.
		validate_all($anvil);
		
		# Call the pull migation.
		($migrated) = $anvil->Server->migrate({
			debug  => 2, 
			server => $server, 
			source => $source,
			target => $target 
		});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { migrated => $migrated }});
	}

	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { migrated => $migrated }});
	if (not $migrated)
	{
		# Exit
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0357"});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# If we made it here, we succeeded.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0360"});
	$anvil->nice_exit({exit_code => 0});
}

# Validation checks that we have the definition XML, resource config and that needed apps are installed.
sub validate_all
{
	my ($anvil) = @_;
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0361"});
	
	### TODO: When we have actual Anvil! systems, connect to the peers (nodes / DR) for this host and see
	###       if the server is running elsewhere.
	
	# Read in an parse the server's XML.
	$anvil->System->check_storage({debug => 3});
	$anvil->Server->get_status({debug => 2, server => $anvil->data->{environment}{OCF_RESKEY_name}});
	
	# Is the name in the definition file what we expect (and did we read the XML data at all)?
	validate_name($anvil);
	
	# Make sure the emulator it wants is the one we have.
	validate_emulator($anvil);
	
	# These tests are only needed if we're about to boot the server
	if (($anvil->data->{switches}{start}) or ($anvil->data->{switches}{migrate_from}))
	{
		# Check that we have enough RAM.
		validate_ram($anvil);
	}
	
	# Validate bridges
	validate_bridges($anvil);
	
	# Validate storage (Disks and optical media)
	validate_storage($anvil);
	
	return(0);
}

# This ensures that the bridges the server connects to exist on this node.
sub validate_bridges
{
	my ($anvil) = @_;
	
	# Get my bridge list
	$anvil->System->get_bridges({debug => 3});
	
	# Find the Optical drives and DRBD devices.
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	foreach my $mac (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{from_disk}{device}{interface}})
	{
		# See if we have this bridge
		my $found  = 0;
		my $bridge = $anvil->data->{server}{'local'}{$server}{from_disk}{device}{interface}{$mac}{bridge};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridge => $bridge }});
		foreach my $interface_name (sort {$a cmp $b} keys %{$anvil->data->{'local'}{network}{bridges}{bridge}})
		{
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { interface_name => $interface_name }});
			if ((exists $anvil->data->{'local'}{network}{bridges}{bridge}{$interface_name}) && ($anvil->data->{'local'}{network}{bridges}{bridge}{$interface_name}{found}))
			{
				$found = 1;
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
				last;
			}
		}
		
		if ($found)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0368", variables => { bridge => $bridge }});
		}
		else
		{
			# Missing bridge.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0369", variables => { bridge => $bridge }});
			$anvil->nice_exit({exit_code => 5});
		}
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0366"});
	
	return(0);
}

# This looks up the disks and optical media connected to this server.
sub validate_storage
{
	my ($anvil) = @_;
	
	### TODO: When checking on a running server, use 'from_memory'.
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	my $source = "from_disk";
	if ($anvil->data->{server}{'local'}{$server}{from_memory}{host})
	{
		$source = "from_memory";
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		server => $server,
		source => $source, 
	}});
	
	### TODO: If we're called with a status and find an ISO file missing and eject it instead of failing.
	###       For now, we just fault out.
	# Do the optical discs in the drive exist? If not, we'll eject it if we're about to boot and fail if
	# we're about to migrate. We skip this check if we're migrating off or shutting down the server.
	if ((exists $anvil->data->{server}{'local'}{$server}{$source}{device}{cdrom}) && (not $anvil->data->{switches}{migrate_to}) && (not $anvil->data->{switches}{stop}))
	{
		foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{$source}{device}{cdrom}{target}})
		{
			if ($anvil->data->{server}{'local'}{$server}{$source}{device}{cdrom}{target}{$device_target}{path})
			{
				my $file = $anvil->data->{server}{'local'}{$server}{$source}{device}{cdrom}{target}{$device_target}{path};
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }});
				if (not -e $file)
				{
					# It doesn't exist. Exit with OCF_ERR_INSTALLED (5).
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0398", variables => { file => $file }});
					$anvil->nice_exit({exit_code => 5});
				}
				elsif (not -r $file)
				{
					# We can't read it. Exit with OCF_ERR_PERM (4).
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0399", variables => { file => $file }});
					$anvil->nice_exit({exit_code => 4});
				}
				else
				{
					# We're OK.
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 2, key => "log_0400", variables => { file => $file }});
				}
			}
		}
	}
	
	# Verify DRBD devices now
	validate_storage_drbd($anvil);
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0367"});
	
	return(0);
}

# THis makes sure that the needed backing DRBD devices are on this node. If so, and if they are not up, they 
# will be brought up. If that fails, it errors out.
sub validate_storage_drbd
{
	my ($anvil) = @_;
	
	# Now check storage.
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	my $source = "from_disk";
	my $host   = $anvil->_short_host_name;
	
	# Did I find a resource for each disk?
	foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{device}})
	{
		next if not $device_path;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			"server::local::${server}::device::${device_path}::resource" => $anvil->data->{server}{'local'}{$server}{device}{$device_path}{resource},
		}});
		if (not $anvil->data->{server}{'local'}{$server}{device}{$device_path}{resource})
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 0, priority => "err", key => "log_0414", variables => { drbd_device => $device_path }});
			$anvil->nice_exit({exit_code => 5});
		}
	}
	
	foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{'local'}{$server}{$source}{device}{disk}{target}})
	{
		my $drbd_device   = $anvil->data->{server}{'local'}{$server}{$source}{device}{disk}{target}{$device_target}{path};
		my $drbd_resource = $anvil->data->{drbd}{config}{$host}{drbd_path}{$drbd_device}{resource};
		my $on_lv         = $anvil->data->{drbd}{config}{$host}{drbd_path}{$drbd_device}{on};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
			host          => $host, 
			drbd_device   => $drbd_device,
			drbd_resource => $drbd_resource, 
			on_lv         => $on_lv, 
		}});
		
		# If the logical volume here here and active?
		if ((not $on_lv) or (not exists $anvil->data->{lvm}{'local'}{lv}{$on_lv}))
		{
			# LV not found
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 0, priority => "err", key => "log_0374", variables => { drbd_device => $drbd_device, lv_path => $on_lv }});
			$anvil->nice_exit({exit_code => 5});
		}
		elsif (not $anvil->data->{lvm}{'local'}{lv}{$on_lv}{active})
		{
			# LV not active. If we're starting the server or we're the migration target, try to 
			# activate it.
			my $active = $anvil->System->activate_lv({debug => 2, path => $on_lv});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active => $active }});
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0413", variables => { lv_path => $on_lv }});
			
			if (not $active)
			{
				# Boo :(
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 0, priority => "err", key => "log_0375", variables => { drbd_device => $drbd_device, lv_path => $on_lv }});
				$anvil->nice_exit({exit_code => 5});
			}
		}
		
		# LV is good if I am still alive.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0376", variables => { 
			drbd_device => $drbd_device,
			lv_path     => $on_lv, 
		}});
		
	}
	
	### NOTE: Checking/Managing firewall ports is expensive option, so DRBD ports are permanently opened 
	###       when a resource is created.
	
	return(0);
}

# This verifies that the requested emulator exists and can be used.
sub validate_emulator
{
	my ($anvil) = @_;
	
	# What emulator is this using?
	my $server   = $anvil->data->{environment}{OCF_RESKEY_name};
	my $emulator = $anvil->data->{server}{'local'}{$server}{from_disk}{info}{emulator};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		emulator                                              => $emulator,
		"server::local::${server}::from_disk::info::emulator" => $anvil->data->{server}{'local'}{$server}{from_disk}{info}{emulator}
	}});
	if (not -e $emulator)
	{
		# It doesn't exist. Exit with OCF_ERR_INSTALLED (5).
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0401", variables => { 
			emulator        => $emulator,
			definition_file => $anvil->data->{server}{definition_file}, 
		}});
		$anvil->nice_exit({exit_code => 5});
	}
	if (not -x $emulator)
	{
		# We can't execute it. Exit with OCF_ERR_PERM (4).
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0402", variables => { emulator => $emulator }});
		$anvil->nice_exit({exit_code => 4});
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0364"});
	
	return(0);
}

# This makes sure the name we see in the definition file matches what we expect.
sub validate_name
{
	my ($anvil) = @_;
	
	my $server = $anvil->data->{environment}{OCF_RESKEY_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		server                                            => $server,
		"server::local::${server}::from_disk::info::name" => $anvil->data->{server}{'local'}{$server}{from_disk}{info}{name}, 
	}});
	
	# If we failed to read the XML, the server probably doesn't exist.
	if (not $anvil->data->{server}{'local'}{$server}{from_disk}{xml})
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0403", variables => { 
			server => $server,
			name   => $anvil->data->{server}{definition_xml}->{name}, 
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	
	# Is the name in the definition file what we expect?
	if ($server ne $anvil->data->{server}{'local'}{$server}{from_disk}{info}{name})
	{
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0403", variables => { 
			server => $server,
			name   => $anvil->data->{server}{'local'}{$server}{from_disk}{info}{name}, 
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0363"});
	
	return(0);
}

# This checks that there is enough RAM to run this server.
sub validate_ram
{
	my ($anvil) = @_;
	
	# How mcuh RAM does the server need and how much do we have free?
	my $server           = $anvil->data->{environment}{OCF_RESKEY_name};
	my $server_ram_bytes = $anvil->data->{server}{'local'}{$server}{from_disk}{memory};
	my $available        = $anvil->System->get_free_memory({debug => 3});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		server_ram_bytes => $anvil->Convert->add_commas({number => $server_ram_bytes})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}).")", 
		available        => $anvil->Convert->add_commas({number => $available})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $available}).")", 
	}});
	if ($server_ram_bytes > $available)
	{
		# Not enough free memory.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0404", variables => { 
			name                => $server, 
			ram                 => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}), 
			ram_bytes           => $anvil->Convert->add_commas({number => $server_ram_bytes}), 
			available_ram       => $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}), 
			available_ram_bytes => $anvil->Convert->add_commas({number => $available}), 
		}});
		$anvil->nice_exit({exit_code => 1});
	}
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0365"});
	
	return(0);
}

### TODO: Make sure the appropriate SN ports are opened.
# This stops (drbdadm down <server>) the storage for a given server on both nodes.
sub manage_drbd_resource
{
	my ($anvil, $task, $resource) = @_;
	
	

	
	return(0);
}

# This reads the XML definition data into an XML data hash.
sub read_server_definition
{
	my ($anvil) = @_;
	
	my $server          =  $anvil->data->{environment}{OCF_RESKEY_name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
		server          => $server, 
	}});
	
	
	return(0);
}

# This logs the details of this call.
sub show_environment
{
	my ($anvil, $level) = @_;
	
	foreach my $key (sort {$a cmp $b} keys %{$anvil->data->{switches}})
	{
		next if $key eq "raw";
		next if $anvil->data->{switches}{$key} eq "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $level, list => { "switches::${key}" => $anvil->data->{switches}{$key} }});
	}
	foreach my $key (sort {$a cmp $b} keys %{$anvil->data->{environment}})
	{
		next if $anvil->data->{environment}{$key} eq "";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $level, list => { "environment::${key}" => $anvil->data->{environment}{$key} }});
	}
	foreach my $key (sort {$a cmp $b} keys %ENV)
	{
		next if exists $anvil->data->{environment}{$key};
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $level, list => { "ENV::${key}" => $ENV{$key} }});
	}
	foreach my $value (@ARGV)
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $level, list => { "ARGV" => $value }});
	}
	
	return(0);
}

# This just prints a quick usage message for now.
sub show_usage
{
	my ($anvil) = @_;
	
	### TODO: How to use this...
	
	$anvil->nice_exit({exit_code => 0});
}

# This prints out the metadata and exits.
sub show_metadata
{
	my ($anvil) = @_;
	
	# This is a pretty simple agent, by design. We only take a server name for now.
	print '<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="ocs:alteeve:server" version="0.1">
  <version>1.0</version>
  <longdesc lang="en">
This resource agent manages KVM+qemu virtual servers on an Anvil! m3 Intelligent Availability(tm) system. 
It manages underlying components like DRBD 9 storage resources, brodge connections and so forth.
  </longdesc>
  <shortdesc lang="en">Anvil! m3 server resource agent</shortdesc>
  <parameters>
    <parameter name="name" unique="1" required="1">
      <longdesc lang="en">
        This is the name of the server as reported by virsh.
      </longdesc>
      <shortdesc lang="en">Server name</shortdesc>
      <content type="string"/>
    </parameter>
  </parameters>
  <actions>
    <action name="start"        timeout="30" />
    <action name="stop"         timeout="60" />
    <action name="monitor"      timeout="10" />
    <action name="notify"       timeout="20" />
    <action name="migrate_to"   timeout="600" />
    <action name="migrate_from" timeout="600" />
    <action name="meta-data"    timeout="5" />
    <action name="validate-all" timeout="20" />
  </actions>
</resource-agent>
';
	
	$anvil->nice_exit({exit_code => 0});
}