From d647014ad117aeb232224d889eb55c36f4291cf3 Mon Sep 17 00:00:00 2001
From: Digimer <digimer@alteeve.ca>
Date: Fri, 7 Aug 2020 23:12:11 -0400
Subject: [PATCH] * Created (finished but not yet tested)
 DRBD->update_global_common() to update DRBD's global_common.conf file. *
 Cleaned up a lot of log levels.

Signed-off-by: Digimer <digimer@alteeve.ca>
---
 Anvil/Tools.pm         |   2 +
 Anvil/Tools/DRBD.pm    | 580 +++++++++++++++++++++++++++++++++++++++++
 share/words.xml        |   1 +
 tools/anvil-join-anvil | 197 +++++++-------
 4 files changed, 688 insertions(+), 92 deletions(-)

diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm
index 9520d3e8..b56b44d7 100644
--- a/Anvil/Tools.pm
+++ b/Anvil/Tools.pm
@@ -1075,6 +1075,7 @@ sub _set_paths
 				'dhcpd.conf'			=>	"/etc/dhcp/dhcpd.conf",
 				'dnf.conf'			=>	"/etc/dnf/dnf.conf",
 				'firewalld.conf'		=>	"/etc/firewalld/firewalld.conf",
+				'global-common.conf'		=>	"/etc/drbd.d/global_common.conf",
 				hosts				=>	"/etc/hosts", 
 				'httpd.conf'			=>	"/etc/httpd/conf/httpd.conf", 
 				'journald_anvil'		=>	"/etc/systemd/journald.conf.d/anvil.conf",
@@ -1164,6 +1165,7 @@ sub _set_paths
 				echo				=>	"/usr/bin/echo",
 				ethtool				=>	"/usr/sbin/ethtool",
 				expect				=>	"/usr/bin/expect", 
+				fence_pacemaker			=>	"/usr/sbin/fence_pacemaker",
 				'firewall-cmd'			=>	"/usr/bin/firewall-cmd",
 				free				=>	"/usr/bin/free",
 				gcc				=>	"/usr/bin/gcc",
diff --git a/Anvil/Tools/DRBD.pm b/Anvil/Tools/DRBD.pm
index 4d193e86..3806c611 100755
--- a/Anvil/Tools/DRBD.pm
+++ b/Anvil/Tools/DRBD.pm
@@ -17,6 +17,8 @@ my $THIS_FILE = "DRBD.pm";
 # get_status
 # manage_resource
 # reload_defaults
+# update_global_common
+# 
 
 =pod
 
@@ -945,6 +947,584 @@ sub reload_defaults
 	return($return_code);
 }
 
+=head2 update_global_common
+
+This configures C<< global_common.conf >> on the local host. Returns C<< !!error!! >> if there is a problem, C<< 0 >> if no update was needed and C<< 1 >> if a change was made.
+
+Parameters;
+
+=head3 usage_count (optional, default '1')
+
+By default, DRBD will call a LINBIT server and add itself to a counter, and then reports back what install number this machine is. This helps LINBIT understand how DRBD is used, and no personal identifiable information is passed to them. If you would like to disable this, set this to C<< 0 >>.
+
+=head3 use_flushes (optional, default '1')
+
+Normally, when a write is done, a flush is called to ensure the data has been written from cache to disk. This is usually desired as it is safe, but does impose a performance penalty.
+
+When there is a hardware RAID controller with protected write cache, explicit flushes can safely be turned off, gaining performance.
+
+If ScanCore can detect a hardware RAID controller, this method will disable disk flushes automatically. This parameter can be used to force flushes on (C<< 1 >>) or off (C<< 0 >>).
+
+B<< Note >>: ScanCore can not yet do this.
+
+=cut
+sub update_global_common
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	
+	my $usage_count = defined $parameter->{usage_count} ? $parameter->{usage_count} : 1;
+	my $use_flushes = defined $parameter->{use_flushes} ? $parameter->{use_flushes} : 0;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		usage_count => $usage_count, 
+		use_flushes => $use_flushes,
+	}});
+	
+	if (not -f $anvil->data->{path}{configs}{'global-common.conf'})
+	{
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0139"});
+		return('!!error!!');
+	}
+	
+	# These values will be used to track where we are in processing the config file and what values are needed.
+	my $update                   = 0;
+	my $usage_count_seen         = 0;
+	my $udev_always_use_vnr_seen = 0;
+	my $fence_peer_seen          = 0;
+	my $auto_promote_seen        = 0;
+	my $disk_flushes_seen        = 0;
+	my $md_flushes_seen          = 0;
+	my $allow_two_primaries_seen = 0;
+	my $after_sb_0pri_seen       = 0;
+	my $after_sb_1pri_seen       = 0;
+	my $after_sb_2pri_seen       = 0;
+	
+	my $in_global   = 0;
+	my $in_common   = 0;
+	my $in_handlers = 0;
+	my $in_startup  = 0;
+	my $in_options  = 0;
+	my $in_disk     = 0;
+	my $in_net      = 0;
+	
+	### NOTE: See 'man drbd.conf-9.0' for details on options.
+	# These values will be used to track where we are in processing the config file and what values are needed.
+	my $say_usage_count         = $usage_count ? "yes" : "no";
+	my $say_fence_peer          = $anvil->data->{path}{exe}{fence_pacemaker};
+	my $say_auto_promote        = "yes";
+	my $say_flushes             = $use_flushes ? "yes" : "no";
+	my $say_allow_two_primaries = "no";
+	my $say_after_sb_0pri       = "discard-zero-changes";
+	my $say_after_sb_1pri       = "discard-secondary";
+	my $say_after_sb_2pri       = "disconnect";
+	
+	# Read in the existing config.
+	my $new_global_common = "";
+	my $old_global_common = $anvil->Storage->read_file({file => $sql_file});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_global_common => $old_global_common }});
+	foreach my $line (split/\n/, $old_global_common)
+	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
+		
+		if ($line =~ /^#/)
+		{
+			$new_global_common .= $line."\n";
+			next;
+		}
+		
+		if ($line =~ /(#.*)$/)
+		{
+			$comment =  $1;
+			$line    =~ s/$comment//;
+		}
+		
+		if ($line =~ /\}/)
+		{
+			if ($in_global)
+			{
+				$in_global = 0;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_global => $in_global }});
+				
+				if (not $usage_count_seen)
+				{
+					$update   = 1;
+					$new_line = "\tusage-count ".$say_usage_count.";";
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					$new_global_common .= $new_line."\n";
+				}
+				if (not $udev_always_use_vnr_seen)
+				{
+					$update   = 1;
+					$new_line = "\tudev-always-use-vnr;";
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					$new_global_common .= $new_line."\n";
+				}
+			}
+			elsif ($in_common)
+			{
+				if ($in_handlers)
+				{
+					$in_handlers = 0;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_handlers => $in_handlers }});
+					
+					if (not $fence_peer_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tfence-peer ".$say_fence_peer.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+				}
+				elsif ($in_startup)
+				{
+					$in_startup = 0;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_startup => $in_startup }});
+					
+					# We don't do anything here yet.
+				}
+				elsif ($in_options)
+				{
+					$in_options = 0;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_options => $in_options }});
+					
+					if (not $auto_promote_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tauto-promote ".$say_auto_promote.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+				}
+				elsif ($in_disk)
+				{
+					$in_disk = 0;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_disk => $in_disk }});
+					
+					if (not $disk_flushes_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tdisk-flushes ".$say_disk_flushes.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+					if (not $md_flushes_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tmd-flushes ".$say_md_flushes.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+				}
+				elsif ($in_net)
+				{
+					$in_net = 0;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_net => $in_net }});
+					
+					if (not $allow_two_primaries_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tallow-two-primaries".$say_allow_two_primaries.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+					
+					if (not $after_sb_0pri_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tafter-sb-0pri".$say_after_sb_0pri.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+					
+					if (not $after_sb_1pri_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tafter-sb-1pri".$say_after_sb_1pri.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+					
+					if (not $after_sb_2pri_seen)
+					{
+						$update   = 1;
+						$new_line = "\t\tafter-sb-2pri".$say_after_sb_2pri.";";
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+							's1:update'   => $update,
+							's2:new_line' => $new_line, 
+						}});
+						$new_global_common .= $new_line."\n";
+					}
+				}
+				else
+				{
+					$in_common = 0;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_common => $in_common }});
+				}
+			}
+			
+			$new_global_common .= $line.$comment."\n";
+			next;
+		}
+		
+		if ($line =~ /global \{/)
+		{
+			$in_global = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { in_global => $in_global }});
+			
+			$new_global_common .= $line.$comment."\n";
+			next;
+		}
+		if ($in_global)
+		{
+			if ($line =~ /(\s*)usage-count(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space       = $1;
+				my $middle_space     = $2;
+				my $value            = $3;
+				my $right_side       = $4;
+				   $usage_count_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'       => $left_space,
+					's2:middle_space'     => $middle_space, 
+					's3:value'            => $value, 
+					's4:right_side'       => $right_side,
+					's5:usage_count_seen' => $usage_count_seen, 
+				}});
+				   
+				if ($value ne $say_usage_count)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."usage-count".$middle_space.$say_usage_count.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+			if ($line =~ /\s*udev-always-use-vnr;$/)
+			{
+				$udev_always_use_vnr_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					udev_always_use_vnr_seen => $usage_count_seen, 
+				}});
+
+				# No change needed
+				$new_global_common .= $line.$comment."\n";
+			}
+		}
+		if ($in_handlers)
+		{
+			if ($line =~ /(\s*)fence-peer(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space       = $1;
+				my $middle_space     = $2;
+				my $value            = $3;
+				my $right_side       = $4;
+				   $fence_peer_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'      => $left_space,
+					's2:middle_space'    => $middle_space, 
+					's3:value'           => $value, 
+					's4:right_side'      => $right_side,
+					's5:fence_peer_seen' => $fence_peer_seen, 
+				}});
+				   
+				if ($value ne $say_fence_peer)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."fence-peer".$middle_space.$say_fence_peer.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+		}
+		if ($in_startup)
+		{
+			# Not doing anything here yet.
+		}
+		if ($in_options)
+		{
+			if ($line =~ /(\s*)auto-promote(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space        = $1;
+				my $middle_space      = $2;
+				my $value             = $3;
+				my $right_side        = $4;
+				   $auto_promote_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'        => $left_space,
+					's2:middle_space'      => $middle_space, 
+					's3:value'             => $value, 
+					's4:right_side'        => $right_side,
+					's5:auto_promote_seen' => $auto_promote_seen, 
+				}});
+				   
+				if ($value ne $say_auto_promote)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."auto-promote".$middle_space.$say_auto_promote.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+		}
+		if ($in_disk)
+		{
+			if ($line =~ /(\s*)disk-flushes(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space        = $1;
+				my $middle_space      = $2;
+				my $value             = $3;
+				my $right_side        = $4;
+				   $disk_flushes_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'        => $left_space,
+					's2:middle_space'      => $middle_space, 
+					's3:value'             => $value, 
+					's4:right_side'        => $right_side,
+					's5:disk_flushes_seen' => $disk_flushes_seen, 
+				}});
+				   
+				if ($value ne $say_flushes)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."disk-flushes".$middle_space.$say_flushes.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+			if ($line =~ /(\s*)md-flushes(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space      = $1;
+				my $middle_space    = $2;
+				my $value           = $3;
+				my $right_side      = $4;
+				   $md_flushes_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'      => $left_space,
+					's2:middle_space'    => $middle_space, 
+					's3:value'           => $value, 
+					's4:right_side'      => $right_side,
+					's5:md_flushes_seen' => $md_flushes_seen, 
+				}});
+				   
+				if ($value ne $say_flushes)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."md-flushes".$middle_space.$say_flushes.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+		}
+		if ($in_net)
+		{
+			if ($line =~ /(\s*)allow-two-primaries(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space        = $1;
+				my $middle_space      = $2;
+				my $value             = $3;
+				my $right_side        = $4;
+				   $allow_two_primaries_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'               => $left_space,
+					's2:middle_space'             => $middle_space, 
+					's3:value'                    => $value, 
+					's4:right_side'               => $right_side,
+					's5:allow_two_primaries_seen' => $allow_two_primaries_seen, 
+				}});
+				   
+				if ($value ne $say_allow_two_primaries)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."allow-two-primaries".$middle_space.$say_allow_two_primaries.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+			if ($line =~ /(\s*)after-sb-0pri(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space        = $1;
+				my $middle_space      = $2;
+				my $value             = $3;
+				my $right_side        = $4;
+				   $after_sb_0pri_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'         => $left_space,
+					's2:middle_space'       => $middle_space, 
+					's3:value'              => $value, 
+					's4:right_side'         => $right_side,
+					's5:after_sb_0pri_seen' => $after_sb_0pri_seen, 
+				}});
+				
+				if ($value ne $say_after_sb_0pri)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."after-sb-0pri".$middle_space.$say_after_sb_0pri.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+			if ($line =~ /(\s*)after-sb-1pr(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space        = $1;
+				my $middle_space      = $2;
+				my $value             = $3;
+				my $right_side        = $4;
+				   $after_sb_1pri_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'         => $left_space,
+					's2:middle_space'       => $middle_space, 
+					's3:value'              => $value, 
+					's4:right_side'         => $right_side,
+					's5:after_sb_1pri_seen' => $after_sb_1pri_seen, 
+				}});
+				
+				if ($value ne $say_after_sb_1pri)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."after-sb-1pri".$middle_space.$say_after_sb_1pri.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+			if ($line =~ /(\s*)after-sb-2pri(\s+)(.*?)(;.*)$/)
+			{
+				my $left_space        = $1;
+				my $middle_space      = $2;
+				my $value             = $3;
+				my $right_side        = $4;
+				   $after_sb_2pri_seen = 1;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					's1:left_space'         => $left_space,
+					's2:middle_space'       => $middle_space, 
+					's3:value'              => $value, 
+					's4:right_side'         => $right_side,
+					's5:after_sb_2pri_seen' => $after_sb_2pri_seen, 
+				}});
+				
+				if ($value ne $say_after_sb_2pri)
+				{
+					   $update   = 1;
+					my $new_line = $left_space."after-sb-2pri".$middle_space.$say_after_sb_2pri.$right_side;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						's1:update'   => $update,
+						's2:new_line' => $new_line, 
+					}});
+					
+					$new_global_common .= $new_line.$comment."\n";
+				}
+				else
+				{
+					# No change needed
+					$new_global_common .= $line.$comment."\n";
+				}
+			}
+		}
+	}
+	
+	return($update);
+}
+
 # =head3
 # 
 # Private Functions;
diff --git a/share/words.xml b/share/words.xml
index e9be2dad..96cf22ca 100644
--- a/share/words.xml
+++ b/share/words.xml
@@ -198,6 +198,7 @@ The error was:
 		<key name="error_0136">System->test_ipmi() was called with an invalid 'lanplus' parameter. It must be 'yes', 'no', 'yes-no' or 'no-yes'. Received: [#!variable!lanplus!#].</key>
 		<key name="error_0137">All attempts to change the IPMI user: [#!variable!user_name!#] (number: [#!variable!user_number!#] failed. The last try's output (if any) was: [#!variable!output!#] (return code: [#!variable!return code!#]).</key>
 		<key name="error_0138">The system call: [#!variable!shell_call!#] failed. The output (if any) was: [#!variable!output!#] (return code: [#!variable!return code!#]).</key>
+		<key name="error_0139">The DRBD global common config file: [#!data!path::configs::global-common.conf!#] doesn't exist, unable to update it.</key>
 		
 		<!-- Table headers -->
 		<key name="header_0001">Current Network Interfaces and States</key>
diff --git a/tools/anvil-join-anvil b/tools/anvil-join-anvil
index 72b16a86..9fddca1b 100755
--- a/tools/anvil-join-anvil
+++ b/tools/anvil-join-anvil
@@ -77,7 +77,8 @@ sub configure_drbd
 {
 	my ($anvil) = @_;
 	
-	
+	### TODO: See if there is a hardware RAID controller and, if so, auto-enable 
+	# Read in the global_common.conf 
 	
 	return(0);
 }
@@ -140,7 +141,7 @@ sub configure_pacemaker
 	my $peer_host_name   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_name : $node1_host_name;
 	my $peer_host_uuid   =  $anvil->Get->host_uuid() eq $node1_host_uuid ? $node2_host_uuid : $node1_host_uuid;
 	my $escaped_password =  shell_quote($new_password);
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 		machine          => $machine, 
 		anvil_uuid       => $anvil_uuid, 
 		anvil_name       => $anvil_name, 
@@ -160,7 +161,7 @@ sub configure_pacemaker
 	if ($machine =~ /dr\d+/)
 	{
 		update_progress($anvil, ($anvil->data->{job}{progress} += 10), "job_0096");
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0096"});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0096"});
 		return(0);
 	}
 	
@@ -173,7 +174,7 @@ sub configure_pacemaker
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
 	$return_code = undef;
 	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0094,!!daemon!pcsd!!");
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0094", variables => { daemon => "pcsd" }});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "pcsd" }});
 	
 	# Disabled and stop the libvirtd daemon.
 	foreach my $daemon ("libvirtd.service", "drbd.service")
@@ -196,11 +197,11 @@ sub configure_pacemaker
 			file   => $anvil->data->{path}{configs}{'corosync.conf'},
 			target => $peer_host_name,
 		});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { corosync_conf => $corosync_conf }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { corosync_conf => $corosync_conf }});
 		if ($corosync_conf ne "!!error!!")
 		{
 			# Write the file out.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0100"});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0100"});
 			$anvil->Storage->write_file({
 				body  => $corosync_conf,
 				file  => $anvil->data->{path}{configs}{'corosync.conf'}, 
@@ -216,7 +217,7 @@ sub configure_pacemaker
 	{
 		my $start_time = 0;
 		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0103");
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0103"});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0103"});
 		
 		# We loop until the peer finishes or the peer's job hit's 100.
 		my $tried_starting = 0;
@@ -229,45 +230,45 @@ sub configure_pacemaker
 				{
 					# Corosync is configured, we'll wait up to two minutes and then try
 					# joining the cluster ourselves.
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0108"});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0108"});
 					$start_time = time + 120;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { start_time => $start_time }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { start_time => $start_time }});
 				}
 				elsif ((time > $start_time) && (not $tried_starting))
 				{
 					# We've waited a minute, time to try starting the cluster.
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0106"});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0106"});
 					my $cluster_started = $anvil->Cluster->start_cluster({debug => 2, all => 1});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cluster_started => $cluster_started }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { cluster_started => $cluster_started }});
 					
 					# Mark that weve tried to start.
 					$tried_starting = 1;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { tried_starting => $tried_starting }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { tried_starting => $tried_starting }});
 				}
 				
 				my $problem = $anvil->Cluster->parse_cib({debug => 3});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { problem => $problem }});
 				if (not $problem)
 				{
 					# See if both nodes are online.
 					my $node1_ready = $anvil->Cluster->check_node_status({node_name => $node1_host_name});
 					my $node2_ready = $anvil->Cluster->check_node_status({node_name => $node2_host_name});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						node1_ready => $node1_ready,
 						node2_ready => $node2_ready, 
 					}});
 					if (($node1_ready) && ($node2_ready))
 					{
 						$both_online = 1;
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { both_online => $both_online }});
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { both_online => $both_online }});
 						
 						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
-						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0104"});
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
 					}
 					else
 					{
 						# Not online yet, wait a bit.
-						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0105", variables => {
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
 							node1_name   => $node1_host_name,
 							node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
 							node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
@@ -285,7 +286,7 @@ sub configure_pacemaker
 			else
 			{
 				# corosync.conf doesn't exist yet.
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0107"});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0107"});
 			}
 			sleep 5 if not $both_online;
 		}
@@ -314,16 +315,16 @@ sub configure_pacemaker
 					# Update the job
 					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0097");
 					$warning_printed = 1;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { warning_printed => $warning_printed }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { warning_printed => $warning_printed }});
 				}
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0097"});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0097"});
 				sleep 5;
 			}
 			else
 			{
 				# We're good.
 				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0098");
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0098"});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0098"});
 				$waiting = 0;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { waiting => $waiting }});
 			}
@@ -335,13 +336,13 @@ sub configure_pacemaker
 		{
 			# There's no cluster yet, initialize it.
 			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0099,!!anvil_name!".$anvil_name."!!");
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0099", variables => { anvil_name => $anvil_name }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0099", variables => { anvil_name => $anvil_name }});
 			
 			my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster setup ".$anvil_name." ".$node1_host_name." ".$node2_host_name;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 			
 			my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 				output      => $output,
 				return_code => $return_code, 
 			}});
@@ -361,7 +362,7 @@ sub configure_pacemaker
 		{
 			# Start the cluster.
 			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0102");
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0102"});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0102"});
 			
 			my $cluster_started = $anvil->Cluster->start_cluster({debug => 3, all => 1});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { cluster_started => $cluster_started }});
@@ -389,12 +390,12 @@ sub configure_pacemaker
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { both_online => $both_online }});
 					
 					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0104");
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0104"});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0104"});
 				}
 				else
 				{
 					# Not online yet, wait a bit.
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0105", variables => {
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0105", variables => {
 						node1_name   => $node1_host_name,
 						node1_ready  => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{in_ccm},
 						node1_in_ccm => $anvil->data->{cib}{parsed}{data}{node}{$node1_host_name}{node_state}{crmd},
@@ -426,7 +427,7 @@ sub configure_pacemaker
 			my $host_uuid         = $node eq "node1" ? $node1_host_uuid : $node2_host_uuid;
 			my $host_ipmi         = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_ipmi};
 			my $ipmi_stonith_name = "ipmilan_".$node; 
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 				node_name         => $node_name, 
 				host_uuid         => $host_uuid, 
 				host_ipmi         => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
@@ -434,11 +435,14 @@ sub configure_pacemaker
 			}});
 			
 			update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0115,!!node!".$node_name."!!");
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0115", variables => { node => $node_name }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0115", variables => { node => $node_name }});
 			
 			# This will store the fence level order. If something changes
 			$fence_order->{$node_name} = [];
 			
+			# This will switch to '1' if something changed, triggering a reconfig of the fencing levels.
+			$something_changed->{$node_name} = 0;
+			
 			# Does this stonith method already exist?
 			my $create_entry    = 0;
 			my $delete_old      = 0;
@@ -451,7 +455,7 @@ sub configure_pacemaker
 				# The --action switch needs to be 'pcmk_off_action' in pcs, so we convert it here.
 				$host_ipmi =~ s/--action status//;
 				$host_ipmi =~ s/--action/--pcmk_off_action/;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					host_ipmi => $host_ipmi =~ /passw/ ? $anvil->Log->is_secure($host_ipmi) : $host_ipmi,
 				}});
 				
@@ -470,7 +474,7 @@ sub configure_pacemaker
 				# If we're here, break up the command and turn it into the pcs call.
 				my $old_switches              = {};
 				my ($fence_agent, $arguments) = ($host_ipmi =~ /^\/.*\/(.*?)\s+(.*)$/);
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					fence_agent  => $fence_agent,
 					arguments    => $arguments =~ /passw/ ? $anvil->Log->is_secure($arguments) : $arguments,
 				}});
@@ -500,7 +504,7 @@ sub configure_pacemaker
 					
 					# Store this to see if it's different from what's already in the CIB.
 					$old_switches->{$argument} = $value;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						"old_switches->{$argument}" => $old_switches->{$argument},
 					}});
 				}
@@ -518,7 +522,7 @@ sub configure_pacemaker
 						next if $argument eq "action";
 						my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}{$argument}{value};
 						my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 							's1:argument'  => $argument, 
 							's2:old_entry' => $old_entry,
 							's3:new_entry' => $new_entry,
@@ -529,13 +533,13 @@ sub configure_pacemaker
 							# Changed, delete and recreate.
 							$delete_old   = 1;
 							$create_entry = 1;
-							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 								delete_old   => $delete_old,
 								create_entry => $create_entry,
 							}});
 							
 							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
-							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0117"});
+							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
 							last;
 						}
 						
@@ -560,17 +564,17 @@ sub configure_pacemaker
 						}});
 						
 						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0117");
-						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0117"});
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0117"});
 					}
 				}
 				else
 				{
 					# No existing entry, add a new one.
 					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0116");
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0116"});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0116"});
 			
 					$create_entry = 1;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { create_entry => $create_entry }});
 				}
 			}
 			elsif (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
@@ -578,14 +582,14 @@ sub configure_pacemaker
 				# There was an existing fence config, but there's no entry in 'host_ipmi'. 
 				# Remove the stonith entry.
 				$delete_old = 1;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { delete_old => $delete_old }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { delete_old => $delete_old }});
 				
 				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0118");
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0118"});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0118"});
 			}
 			
 			# Process the IPMI entry.
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 				delete_old   => $delete_old,
 				create_entry => $create_entry, 
 			}});
@@ -593,13 +597,13 @@ sub configure_pacemaker
 			{
 				# Delete
 				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$ipmi_stonith_name."!!");
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0119", variables => { device => $ipmi_stonith_name }});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $ipmi_stonith_name }});
 				
 				my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$ipmi_stonith_name;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 				
 				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					output      => $output,
 					return_code => $return_code, 
 				}});
@@ -617,19 +621,19 @@ sub configure_pacemaker
 				}
 				
 				$something_changed->{$node_name} = 1;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
 			}
 			if ($create_entry)
 			{
 				# Create.
 				update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$ipmi_stonith_name."!!");
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0120", variables => { device => $ipmi_stonith_name }});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $ipmi_stonith_name }});
 				
 				my $shell_call = $pcs_add_command;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 				
 				my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					output      => $output,
 					return_code => $return_code, 
 				}});
@@ -647,7 +651,7 @@ sub configure_pacemaker
 				}
 				
 				$something_changed->{$node_name} = 1;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
 			}
 			
 			
@@ -663,7 +667,7 @@ sub configure_pacemaker
 				my $fence_agent     = $anvil->data->{fences}{fence_uuid}{$fence_uuid}{fence_agent};
 				my $stonith_name    = ($fence_agent =~ /^fence_(.*)$/)[0]."_".$node."_".$fence_name; 
 				my $port            = $anvil->data->{manifests}{manifest_uuid}{$manifest_uuid}{parsed}{machine}{$node}{fence}{$device}{port};
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					device          => $device, 
 					fence_uuid      => $fence_uuid, 
 					fence_name      => $fence_name, 
@@ -683,20 +687,20 @@ sub configure_pacemaker
 				
 				# Fence arguments use 'action', but pcs deprecated it in favour of 'pcmk_off_action', so rename it.
 				$fence_arguments =~ s/action=/pcmk_off_action=/;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					fence_arguments => $fence_arguments =~ /passw/ ? $anvil->Log->is_secure($fence_arguments) : $fence_arguments, 
 				}});
 				
 				# Build the pcs command
 				my $pcs_add_command = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." ".$fence_agent." pcmk_host_list=\"".$node_name."\" ".$fence_arguments." ";
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
 				}});
 				foreach my $pair (split/ /, $fence_arguments)
 				{
 					# Ignore 'delay', we handle that in Cluster->set_delay();
 					my ($argument, $value) = ($pair =~ /^(.*)="(.*?)"$/);
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						pair     => $pair =~ /passw/ ? $anvil->Log->is_secure($pair) : $pair,
 						argument => $argument,
 						value    => $argument =~ /passw/ ? $anvil->Log->is_secure($value) : $value,
@@ -708,7 +712,7 @@ sub configure_pacemaker
 					
 					# Store this to see if it's different from what's already in the CIB.
 					$old_switches->{$argument} = $value;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						"old_switches->{$argument}" => $old_switches->{$argument},
 					}});
 				}
@@ -717,13 +721,13 @@ sub configure_pacemaker
 					$port                 =~ s/"/\\"/g;
 					$pcs_add_command      .= "port=\"".$port."\" ";
 					$old_switches->{port} =  $port;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
 						"old_switches->{port}" => $old_switches->{port},
 					}});
 				}
 				$pcs_add_command .= "op monitor interval=\"60\"";
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command, 
 				}});
 				
@@ -735,7 +739,7 @@ sub configure_pacemaker
 						next if $argument eq "delay";
 						my $old_entry = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name}{argument}{$argument}{value};
 						my $new_entry = exists $old_switches->{$argument} ? $old_switches->{$argument} : "";
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 							's1:argument'  => $argument, 
 							's2:old_entry' => $old_entry,
 							's3:new_entry' => $new_entry,
@@ -746,13 +750,13 @@ sub configure_pacemaker
 							# Changed, delete and recreate.
 							$delete_old   = 1;
 							$create_entry = 1;
-							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+							$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 								delete_old   => $delete_old,
 								create_entry => $create_entry,
 							}});
 							
 							update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
-							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0121", variables => { device => $stonith_name }});
+							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
 							last;
 						}
 						
@@ -762,42 +766,46 @@ sub configure_pacemaker
 					
 					# Are there any old switches left?
 					my $old_switch_count = keys %{$old_switches};
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_switch_count => $old_switch_count }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_switch_count => $old_switch_count }});
 					if ((not $delete_old) && ($old_switch_count))
 					{
 						# Delete and recreate. 
 						$delete_old   = 1;
 						$create_entry = 1;
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 							delete_old   => $delete_old,
 							create_entry => $create_entry,
 						}});
 						
 						update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0121,!!device!".$stonith_name."!!");
-						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0121", variables => { device => $stonith_name }});
+						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0121", variables => { device => $stonith_name }});
 					}
 				}
 				else
 				{
 					# No existing entry, add a new one.
 					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0122,!!device!".$stonith_name."!!");
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0122", variables => { device => $stonith_name }});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0122", variables => { device => $stonith_name }});
 					
 					$create_entry = 1;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_entry => $create_entry }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { create_entry => $create_entry }});
 				}
 				
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+					delete_old   => $delete_old,
+					create_entry => $create_entry, 
+				}});
 				if ($delete_old)
 				{
 					# Delete
 					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0119,!!device!".$stonith_name."!!");
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0119", variables => { device => $stonith_name }});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0119", variables => { device => $stonith_name }});
 					
 					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith delete ".$stonith_name;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 					
 					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						output      => $output,
 						return_code => $return_code, 
 					}});
@@ -815,19 +823,19 @@ sub configure_pacemaker
 					}
 					
 					$something_changed->{$node_name} = 1;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
 				}
 				if ($create_entry)
 				{
 					# Create.
 					update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0120,!!device!".$stonith_name."!!");
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0120", variables => { device => $stonith_name }});
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $stonith_name }});
 					
 					my $shell_call = $pcs_add_command;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 					
 					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						output      => $output,
 						return_code => $return_code, 
 					}});
@@ -845,12 +853,17 @@ sub configure_pacemaker
 					}
 					
 					$something_changed->{$node_name} = 1;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
 				}
 			}
 			
 			
 			### If we had a fence_ipmilan entry, add a 'fence_delay' entry, if needed.
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+				node            => $node, 
+				node1_use_delay => $node1_use_delay,
+				node2_use_delay => $node2_use_delay, 
+			}});
 			if ((($node eq "node1") && ($node1_use_delay)) or 
 			    (($node eq "node2") && ($node2_use_delay)))
 			{
@@ -862,10 +875,10 @@ sub configure_pacemaker
 				if (not exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$stonith_name})
 				{
 					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith create ".$stonith_name." fence_delay pcmk_host_list=\"".$node_name."\" wait=\"60\" op monitor interval=\"60\"";
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 					
 					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						output      => $output,
 						return_code => $return_code, 
 					}});
@@ -888,7 +901,7 @@ sub configure_pacemaker
 		# Setup fence levels.
 		foreach my $node_name (sort {$a cmp $b} keys %{$fence_order})
 		{
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
 			if ($something_changed->{$node_name})
 			{
 				# Delete any existing fence levels
@@ -898,10 +911,10 @@ sub configure_pacemaker
 					{
 						# pcs stonith level delete <index> <target>
 						my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level delete ".$index." ".$node_name;
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 						
 						my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 							output      => $output,
 							return_code => $return_code, 
 						}});
@@ -927,17 +940,17 @@ sub configure_pacemaker
 					my $devices = "";
 					foreach my $device (sort {$a cmp $b} @{$fence_devices->{$node_name}{$fence_agent}})
 					{
-						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device => $device }});
+						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { device => $device }});
 						$devices .= $device.",";
 					}
 					$devices =~ s/,$//;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { devices => $devices }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { devices => $devices }});
 					
 					my $shell_call = $anvil->data->{path}{exe}{pcs}." stonith level add ".$i." ".$node_name." ".$devices;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 					
 					my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 						output      => $output,
 						return_code => $return_code, 
 					}});
@@ -961,19 +974,19 @@ sub configure_pacemaker
 	}
 	
 	# Enable fencing and set the retry to INFINITY, if needed.
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 		"cib::parsed::data::stonith::max-attempts" => $anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'}, 
 	}});
 	if ($anvil->data->{cib}{parsed}{data}{stonith}{'max-attempts'} ne "INFINITY")
 	{
 		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0124");
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0124"});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0124"});
 		
 		my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-max-attempts=INFINITY";
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 		
 		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 			output      => $output,
 			return_code => $return_code, 
 		}});
@@ -991,19 +1004,19 @@ sub configure_pacemaker
 		}
 	}
 	
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 		"cib::parsed::data::stonith::enabled" => $anvil->data->{cib}{parsed}{data}{stonith}{enabled}, 
 	}});
 	if (not $anvil->data->{cib}{parsed}{data}{stonith}{enabled})
 	{
 		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0125");
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0125"});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0125"});
 		
 		my $shell_call = $anvil->data->{path}{exe}{pcs}." property set stonith-enabled=true";
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 		
 		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 			output      => $output,
 			return_code => $return_code, 
 		}});
@@ -1079,12 +1092,12 @@ sub check_local_network
 	{
 		# No need to change
 		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0077,!!host_name!".$new_host_name."!!");
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0077", variables => { host_name => $new_host_name }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0077", variables => { host_name => $new_host_name }});
 	}
 	
 	# Read the local network manager data.
 	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0080");
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0080"});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0080"});
 	$anvil->Network->read_nmcli({debug => 3});
 	
 	# Now check IP addresses.
@@ -1607,7 +1620,7 @@ sub check_local_network
 	$anvil->Network->read_nmcli({debug => 3});
 	$anvil->Network->get_ips({debug => 3});
 	
-	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "job_0086"});
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0086"});
 	update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0086");
 	
 	# Update MTUs (running interface and config) if needed.