* Fairly heavy rework of ocf:alteeve:server to update how it handles storage during server start. It now properly handles the new "one resource, multiple volumes" and "start resource, not daemon" approach.

Signed-off-by: Digimer <digimer@alteeve.ca>
6 years ago · 3cf1ad2ff8
parent 7e4a170382
commit 3cf1ad2ff8
2 changed files with 284 additions and 204 deletions
--- a/ocf/alteeve/server
+++ b/ocf/alteeve/server
@ -9,6 +9,11 @@
 #          cluster or on any configuration outside how the Anvil! m3 uses it. If you plan to adapt it to 
 #          another purpose, let us know and we'll try to help.
 # 
+# NOTE: This was initially written with the idea that multiple resources could be used by a single server. 
+#       Now. we use a single resource, named after the server, with 1 or more volumes per resource. As such,
+#       you will see (for now) an attempt to parse resources, which is not needed and will be removed in 
+#       time.
+# 
 # Based on: https://github.com/ClusterLabs/resource-agents/blob/master/doc/dev-guides/ra-dev-guide.asc
 # 
 # Error types from pacemaker's perspective;
@ -155,12 +160,14 @@ if ($anvil->data->{switches}{test3})
 {
 	$anvil->data->{switches}{start}                          = "#!set!#";
 	$anvil->data->{environment}{OCF_RESKEY_name}             = "test_server";
+	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node} = "el8-a01n01.digimer.ca";
 	print "Running test 3; Boot: [".$anvil->data->{environment}{OCF_RESKEY_name}."] locally.\n";
 }
 if ($anvil->data->{switches}{test4})
 {
 	$anvil->data->{switches}{stop}                           = "#!set!#";
 	$anvil->data->{environment}{OCF_RESKEY_name}             = "test_server";
+	$anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node} = "el8-a01n01.digimer.ca";
 	print "Running test 3; Shut down: [".$anvil->data->{environment}{OCF_RESKEY_name}."] locally.\n";
 }

@ -177,7 +184,7 @@ if (($anvil->data->{switches}{monitor})     or
 }
 else
 {
-	show_environment($anvil, 2);
+	show_environment($anvil, 3);
 }

 ### What are we being asked to do?
@ -250,7 +257,7 @@ elsif ($anvil->data->{switches}{notify})
 else
 {
 	# We were called in some unexpected way. Log an error, show usage and exit.
-	show_environment($anvil, 0);
+	show_environment($anvil, 3);
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level =>0, key => "log_0302"});
 	$anvil->nice_exit({exit_code => 1});
 }
@ -331,6 +338,8 @@ sub start_server
 				'state' => $state,
 			}});
 			
+			# Make sure the server is shut down, if it is listed at all. Any other state is 
+			# unexpected and needs to be sorted by a human.
 			if ($state ne "shut down")
 			{
 				# Abort
@ -612,7 +621,31 @@ sub stop_server
 			
 			# Stop DRBD resources now. We don't worry if it actually stops or not (let ScanCore 
 			# handle that). We only care that the server has stopped.
-			manage_drbd_resource($anvil, "down");
+			foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{disks}})
+			{
+				my $resource = $anvil->data->{device_path}{$device_path}{resource};
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+					device_path => $device_path, 
+					resource    => $resource,
+				}});
+				
+				if ((not exists $anvil->data->{drbd}{stopped}{$resource}) or (not $anvil->data->{drbd}{stopped}{$resource}))
+				{
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0408", variables => { 
+						resource    => $resource,
+						device_path => $device_path,
+					}});
+					manage_drbd_resource($anvil, "down", $resource);
+					$anvil->data->{drbd}{stopped}{$resource} = 1;
+				}
+				else
+				{
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0409", variables => { 
+						resource    => $resource,
+						device_path => $device_path,
+					}});
+				}
+			}
 			$anvil->nice_exit({exit_code => 0});
 		}
 		
@ -1120,7 +1153,7 @@ sub validate_bridges
 	{
 		if ($anvil->data->{'local'}{bridge}{$bridge})
 		{
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0368", variables => { bridge => $bridge }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0368", variables => { bridge => $bridge }});
 		}
 		else
 		{
@ -1144,7 +1177,7 @@ sub validate_storage
 		foreach my $disk_ref (@{$device_ref->{disk}})
 		{
 			my $type = $disk_ref->{device};
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { type => $type }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }});
 			if ($type eq "disk")
 			{
 				foreach my $source_ref (@{$disk_ref->{source}})
@ -1160,7 +1193,7 @@ sub validate_storage
 				{
 					my $file                                  = $source_ref->{file};
 					   $anvil->data->{server}{optical}{$file} = 1;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "server::optical::${file}" => $anvil->data->{server}{optical}{$file} }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "server::optical::${file}" => $anvil->data->{server}{optical}{$file} }});
 				}
 			}
 		}
@ -1215,13 +1248,14 @@ sub validate_storage_drbd
 	{
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
 		
+		# Figure out who I am and who my peer is, ignoring DR host(s).
 		my $peer  = "";
 		my $local = "";
 		foreach my $connection_ref (@{$drbd_xml->{resource}->{$resource}->{connection}})
 		{
 			my $protocol = $connection_ref->{section}->{net}->{option}->{protocol}->{value};
 			my $fencing  = $connection_ref->{section}->{net}->{option}->{fencing}->{value};
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 				protocol => $protocol, 
 				fencing  => $fencing, 
 			}});
@ -1238,7 +1272,7 @@ sub validate_storage_drbd
 				my $short_hostname =  $host;
 				   $short_hostname =~ s/\..*$//;
 				my $local_hostname = $anvil->data->{environment}{OCF_RESKEY_CRM_meta_on_node};
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 					host           => $host, 
 					short_hostname => $short_hostname, 
 					address        => $address, 
@ -1251,7 +1285,7 @@ sub validate_storage_drbd
 				{
 					# This is us.
 					$local = $host;
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0371", variables => { 
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0371", variables => { 
 						resource => $resource,
 						address  => $address, 
 						port     => $port, 
@ -1264,13 +1298,13 @@ sub validate_storage_drbd
 					# Record my node name for this resource (to be paired with the node 
 					# ID when migrating)
 					$anvil->data->{resource}{$resource}{local_node_name} = $host;
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "resource::${resource}::local_node_name" => $anvil->data->{resource}{$resource}{local_node_name} }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "resource::${resource}::local_node_name" => $anvil->data->{resource}{$resource}{local_node_name} }});
 				}
 				else
 				{
 					# This is our peer
 					$peer = $host;
-					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0372", variables => { 
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0372", variables => { 
 						resource => $resource,
 						address  => $address, 
 						port     => $port, 
@ -1283,12 +1317,13 @@ sub validate_storage_drbd
 			}
 		}
 		
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 			'local' => $local, 
 			 peer   => $peer, 
 		}});
 		foreach my $volume (sort {$a cmp $b} keys %{$drbd_xml->{resource}->{$resource}->{host}->{$local}->{volume}})
 		{
+			# The backing device is the logical volume underpinning this DRBD device on this node.
 			my $backing_device = $drbd_xml->{resource}->{$resource}->{host}->{$local}->{volume}->{$volume}->{disk}->[0];
 			my $device_path    = $drbd_xml->{resource}->{$resource}->{host}->{$local}->{volume}->{$volume}->{device}->[0]->{content};
 			my $device_minor   = $drbd_xml->{resource}->{$resource}->{host}->{$local}->{volume}->{$volume}->{device}->[0]->{minor};
@ -1301,38 +1336,39 @@ sub validate_storage_drbd
 			
 			$anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv}    = $backing_device;
 			$anvil->data->{server}{drbd}{'local'}{device}{$device_path}{minor} = $device_minor;
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 				"server::drbd::local::device::${device_path}::lv"    => $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv},
 				"server::drbd::local::device::${device_path}::minor" => $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{minor}, 
 			}});
 			
 			# Map the resource name to the local drbd device path.
-			$anvil->data->{resource}{$resource}{lv}             = $backing_device;
-			$anvil->data->{resource}{$resource}{path}           = $device_path;
+			$anvil->data->{resource}{$resource}{volume}{$volume}{lv}   = $backing_device;
+			$anvil->data->{resource}{$resource}{volume}{$volume}{path} = $device_path;
 			$anvil->data->{device_path}{$device_path}{resource}        = $resource;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-				"resource::${resource}::path"           => $anvil->data->{resource}{$resource}{path},
-				"resource::${resource}::lv"             => $anvil->data->{resource}{$resource}{lv}, 
+				"resource::${resource}::volume::${volume}::path" => $anvil->data->{resource}{$resource}{volume}{$volume}{path},
+				"resource::${resource}::volume::${volume}::lv"   => $anvil->data->{resource}{$resource}{volume}{$volume}{lv}, 
 				"device_path::${device_path}::resource"          => $anvil->data->{device_path}{$device_path}{resource}, 
 			}});
 		}
 	}

+	# Pair the volumes to their backing LVs.
 	foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{disks}})
 	{
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0373", variables => { device_path => $device_path }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 3, key => "log_0373", variables => { device_path => $device_path }});
 		
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "server::drbd::local::device::${device_path}::lv" => $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv} }});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "server::drbd::local::device::${device_path}::lv" => $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv} }});
 		if (not $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv})
 		{
 			# The backing LV doesn't exist.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0374", variables => { device_path => $device_path }});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 0, priority => "err", key => "log_0374", variables => { device_path => $device_path }});
 			$anvil->nice_exit({exit_code => 5});
 		}
 		elsif (not -e $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv})
 		{
 			# The backing LV doesn't exist.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0375", variables => { 
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 0, priority => "err", key => "log_0375", variables => { 
 				device_path => $device_path,
 				lv          => $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv}, 
 			}});
@ -1340,7 +1376,7 @@ sub validate_storage_drbd
 		}
 		else
 		{
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0376", variables => { 
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0376", variables => { 
 				device_path => $device_path,
 				lv          => $anvil->data->{server}{drbd}{'local'}{device}{$device_path}{lv}, 
 			}});
@ -1356,6 +1392,11 @@ sub validate_storage_drbd
 	# Now read in the status of the drbd devices
 	$return_code                    = undef;
 	(my $status_json, $return_code) = $anvil->System->call({shell_call =>  $anvil->data->{path}{exe}{drbdsetup}." status --json"});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+		status_json => $status_json,
+		return_code => $return_code, 
+		json_length => length($status_json), 
+	}});
 	if ($return_code)
 	{
 		# Something went wrong.
@ -1367,39 +1408,50 @@ sub validate_storage_drbd
 	}
 	
 	# If DRBD is not up, the returned JSON output will not actually exist.
-	if ($status_json =~ /No currently configured DRBD found/si)
+	if (($status_json =~ /No currently configured DRBD found/si) or (not check_drbd_status($anvil, $status_json)))
 	{
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0378"});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 2, key => "log_0378"});
 		foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{disks}})
 		{
 			my $resource = $anvil->data->{device_path}{$device_path}{resource};
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0379", variables => { 
-				resource    => $resource,
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 				device_path => $device_path, 
+				resource    => length($resource), 
 			}});
 			
-			(my $drbdadm_output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." up $resource"});
-			if ($return_code)
+			if ((not exists $anvil->data->{drbd}{started}{$resource}) or (not $anvil->data->{drbd}{started}{$resource}))
 			{
-				# Something went wrong.
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0380", variables => { 
-					return_code    => $return_code,
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0379", variables => { 
 					resource    => $resource,
-					drbdadm_output => $drbdadm_output, 
+					device_path => $device_path,
 				}});
-				$anvil->nice_exit({exit_code => 1});
 			}
+			else
+			{
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0407", variables => { 
+					resource    => $resource,
+					device_path => $device_path,
+				}});
+				next;
+			}
+			
+			manage_drbd_resource($anvil, "up", $resource);
+			$anvil->data->{drbd}{started}{$resource} = 1;
 		}
 		
 		# Give them a few seconds to start.
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0381"});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0381"});
 		sleep 3;
 		
 		# Check DRBD setup again
 		$return_code = undef;
 		$status_json = undef;
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0385"});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0385"});
 		($status_json, $return_code) = $anvil->System->call({shell_call =>  $anvil->data->{path}{exe}{drbdsetup}." status --json"});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+			status_json => $status_json,
+			return_code => $return_code, 
+		}});
 		if ($return_code)
 		{
 			# Something went wrong.
@ -1411,7 +1463,7 @@ sub validate_storage_drbd
 		}
 		
 		# If DRBD is still not up, we're done.
-		if ($status_json =~ /No currently configured DRBD found/si)
+		if (($status_json =~ /No currently configured DRBD found/si) or (length($status_json) < 5))
 		{
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0383"});
 			$anvil->nice_exit({exit_code => 1});
@ -1419,12 +1471,20 @@ sub validate_storage_drbd
 	}
 	
 	# Process the JSON data. If any disks are not seen, they won't be set to 'ok', which we'll catch next.
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { status_json => $status_json }});
 	check_drbd_status($anvil, $status_json);
 	
+	### NOTE: The checks below might no longer be needed.
+=cut
 	# Make sure I saw all disks.
 	my $check_again = 0;
 	foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{disks}})
 	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
+			device_path                     => $device_path,
+			"server::disks::${device_path}" => $anvil->data->{server}{disks}{$device_path}, 
+		}});
+		
 		if ($anvil->data->{server}{disks}{$device_path} eq "check")
 		{
 			# Failed to see it, see if we can bring it up. 
@ -1435,6 +1495,7 @@ sub validate_storage_drbd
 				device_path => $device_path,
 			}});
 			
+			#manage_drbd_resource($anvil, "up", $resource);
 			(my $drbdadm_output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." up $resource"});
 			if ($return_code)
 			{
@ -1476,6 +1537,7 @@ sub validate_storage_drbd
 	}
 	
 	# Do I need to check again?
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {check_again => $check_again }});
 	if ($check_again)
 	{
 		foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{disks}})
@ -1492,8 +1554,9 @@ sub validate_storage_drbd
 			}
 		}
 	}
+=cut
 	
-	### TODO: Finish this.
+	### TODO: Finish this, whatever this was going to be...
 	# If I am about to push a server off, we need to make sure the peer is UpToDate
 	if ($anvil->data->{switches}{migrate_to})
 	{
@ -1510,31 +1573,37 @@ sub validate_storage_drbd
 sub check_drbd_status
 {
 	my ($anvil, $status_json) = @_;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { status_json => $status_json }});
 	
 	my $json           = JSON->new->allow_nonref;
 	my $drbd_status    = $json->decode($status_json);
+	my $resource_found = 0;
 	foreach my $resource_ref (@{$drbd_status})
 	{
 		my $resource = $resource_ref->{name};
-		my $device_path    = $anvil->data->{resource}{$resource}{path};
-		my $logical_volume = $anvil->data->{resource}{$resource}{lv};
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			resource       => $resource, 
-			device_path    => $device_path, 
-			logical_volume => $logical_volume, 
-		}});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { resource => $resource }});
 		
 		# Record my node ID for this resource
 		$anvil->data->{resource}{$resource}{local_node_id} = $resource_ref->{'node-id'};
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "resource::${resource}::local_node_id" => $anvil->data->{resource}{$resource}{local_node_id} }});
 		
+		foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{resource}{$resource}{volume}})
+		{
+			my $device_path    = $anvil->data->{resource}{$resource}{volume}{$volume}{path};
+			my $logical_volume = $anvil->data->{resource}{$resource}{volume}{$volume}{lv};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				's1:volume'         => $volume, 
+				's2:device_path'    => $device_path, 
+				's3:logical_volume' => $logical_volume, 
+			}});
+		
 			if ((exists $anvil->data->{server}{disks}{$device_path}) && ($anvil->data->{server}{disks}{$device_path} eq "check"))
 			{
 				### This disk is in use by this server, check it.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0388", variables => { device_path => $device_path }});
+				$resource_found = 1;
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 2, key => "log_0388", variables => { device_path => $device_path }});
 				
-			# If we're booting a server or migrating it here, we need to make sure all local 
-			# volumes are UpToDate?
+				# We can't run the server here until our device(s) are UpToDate or SyncSource.
 				if (($anvil->data->{switches}{start}) or ($anvil->data->{switches}{migrate_from}))
 				{
 					foreach my $device_ref (@{$resource_ref->{devices}})
@ -1542,7 +1611,7 @@ sub check_drbd_status
 						# Are we UpToDate (or SyncSource)?
 						if ((lc($device_ref->{'disk-state'}) ne "uptodate") && (lc($device_ref->{'disk-state'}) ne "syncsource"))
 						{
-						# We can't start here.
+							# We can't run here.
 							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0389", variables => { 
 								resource   => $resource,
 								volume     => $device_ref->{volume}, 
@ -1552,7 +1621,8 @@ sub check_drbd_status
 						}
 						else
 						{
-						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0390", variables => { 
+							# We're good.
+							$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 2, key => "log_0390", variables => { 
 								resource   => $resource, 
 								volume     => $device_ref->{volume}, 
 								disk_state => $device_ref->{'disk-state'}, 
@ -1561,28 +1631,29 @@ sub check_drbd_status
 					}
 				}
 				
-			# If we're booting a server, we need to be sure that *no* peer is Primary. If we're 
-			# migrating, we need to be sure the migration target is UpToDate.
+				# If we're booting a server, we need to be sure that *no* peer is Primary.
 				foreach my $connection_ref (@{$resource_ref->{connections}})
 				{
+					# If we're not connected, skip.
+					my $connection_state = $connection_ref->{'connection-state'};
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { connection_state => $connection_state }});
+					next if lc($connection_state) ne "connected";
+					
 					# Is the peer's role Primary? In all cases, we abort if so.
-				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0390", variables => { name => $connection_ref->{name} }});
-				if (lc($connection_ref->{'peer-role'}) eq "primary")
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 2, key => "log_0391", variables => { name => $connection_ref->{name} }});
+					if ((lc($connection_ref->{'peer-role'}) eq "primary") && ($anvil->data->{switches}{start}))
 					{
 						# Don't boot here
-					if ($anvil->data->{switches}{start})
-					{
 						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0392", variables => { 
 							resource => $resource,
 							name     => $connection_ref->{name}
 						}});
 						$anvil->nice_exit({exit_code => 1});
 					}
-				}
 					
 					# If we're migrating to the peer, make sure the target disk state is UpToDate
 					# or SyncSource.
-				if (($anvil->data->{switches}{migrate_to}) or ($anvil->data->{switches}{migrate_to}))
+					if ($anvil->data->{switches}{migrate_to})
 					{
 						# Is this connection to our migration target?
 						my $peer_short_name  =  $connection_ref->{name};
@ -1601,7 +1672,6 @@ sub check_drbd_status
 						}
 						
 						# We will need the node ID to enable dual-primary.
-					#print Dumper $connection_ref;
 						$anvil->data->{resource}{$resource}{target_name}    = $connection_ref->{name};
 						$anvil->data->{resource}{$resource}{target_node_id} = $connection_ref->{'peer-node-id'};
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
@ -1635,11 +1705,39 @@ sub check_drbd_status
 			else
 			{
 				# Ignoring, not used.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0396", variables => { device_path => $device_path }});
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 2, key => "log_0396", variables => { device_path => $device_path }});
 			}
 		}
 		
-	return(0);
+		# If we're still alive and we're booting a server, make sure the local resource is Primary
+		my $role = $resource_ref->{role};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { role => $role }});
+		if (($anvil->data->{switches}{start}) && (lc($role) eq "secondary"))
+		{
+			# Go primary.
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 0, level => 1, key => "log_0410", variables => { 
+				resource => $resource,
+				role     => $role,
+			}});
+			my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." primary ".$resource});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				output      => $output, 
+				return_code => $return_code, 
+			}});
+			if ($return_code)
+			{
+				# Something went wrong
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0411", variables => { 
+					resource    => $resource,
+					return_code => $return_code,
+					output      => $output, 
+				}});
+				$anvil->nice_exit({exit_code => 1});
+			}
+		}
+	}
+	
+	return($resource_found);
 }

 # This makes sure that any media in the server's optical drive exists here and is readable.
@ -1649,7 +1747,7 @@ sub validate_storage_optical
 	
 	foreach my $file (sort {$a cmp $b} keys %{$anvil->data->{server}{optical}})
 	{
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0397", variables => { file => $file }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0397", variables => { file => $file }});
 		
 		# If the file doesn't exist, exit with OCF_ERR_INSTALLED (5). If we can't read it, exit with 
 		# OCF_ERR_PERM (4).
@ -1682,7 +1780,7 @@ sub validate_emulator
 	
 	# What emulator is this using?
 	my $emulator = $anvil->data->{server}{definition_xml}->{devices}->[0]->{emulator}->[0];
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { emulator => $emulator }});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { emulator => $emulator }});
 	if (not -e $emulator)
 	{
 		# It doesn't exist. Exit with OCF_ERR_INSTALLED (5).
@ -1728,22 +1826,16 @@ sub validate_ram
 	# How mcuh RAM does the server need?
 	my $server_ram_value = $anvil->data->{server}{definition_xml}->{memory}->[0]->{content};
 	my $server_ram_units = $anvil->data->{server}{definition_xml}->{memory}->[0]->{unit};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
 		server_ram_value => $server_ram_value, 
 		server_ram_units => $server_ram_units, 
 	}});
 	
 	# Convert to bytes
-	my $server_ram_bytes = $server_ram_value;
-	if    ($server_ram_units =~ /^k/i) { $server_ram_bytes = ($server_ram_value * (2 ** 10));                             }
-	elsif ($server_ram_units =~ /^m/i) { $server_ram_bytes = ($server_ram_value * (2 ** 20));                             }
-	elsif ($server_ram_units =~ /^g/i) { $server_ram_bytes = ($server_ram_value * (2 ** 30));                             }
-	elsif ($server_ram_units =~ /^t/i) { $server_ram_bytes = ($server_ram_value * (2 ** 40));                             }
-	elsif ($server_ram_units =~ /^p/i) { $server_ram_bytes = Math::BigInt->new('2')->bpow('50')->bmul($server_ram_value); }
-	elsif ($server_ram_units =~ /^e/i) { $server_ram_bytes = Math::BigInt->new('2')->bpow('60')->bmul($server_ram_value); }
-	elsif ($server_ram_units =~ /^z/i) { $server_ram_bytes = Math::BigInt->new('2')->bpow('70')->bmul($server_ram_value); }
-	elsif ($server_ram_units =~ /^y/i) { $server_ram_bytes = Math::BigInt->new('2')->bpow('80')->bmul($server_ram_value); }
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_ram_bytes => $server_ram_bytes }});
+	my $server_ram_bytes = $anvil->Convert->human_readable_to_bytes({size => $server_ram_value, type => $server_ram_units, base2 => 1 });
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+		server_ram_bytes => $server_ram_bytes." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}).")",
+	}});
 	
 	# How much RAM do we have available?
 	my $available               = 0;
@ -1760,19 +1852,19 @@ sub validate_ram
 			my $cache     = $5;
 			   $available = $6;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
-				total     => $total, 
-				used      => $used,
-				free      => $free, 
-				shared    => $shared, 
-				cache     => $cache, 
-				available => $available, 
+				total     => $total." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $total})."})", 
+				used      => $used." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $used})."})",
+				free      => $free." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $free})."})", 
+				shared    => $shared." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $shared})."})", 
+				cache     => $cache." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $cache})."})", 
+				available => $available." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $available})."})", 
 			}});
 		}
 	}
 	
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-		server_ram_bytes => $anvil->Convert->add_commas({number => $server_ram_bytes}), 
-		available        => $anvil->Convert->add_commas({number => $available}), 
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+		server_ram_bytes => $anvil->Convert->add_commas({number => $server_ram_bytes})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram_bytes}).")", 
+		available        => $anvil->Convert->add_commas({number => $available})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $available}).")", 
 	}});
 	if ($server_ram_bytes > $available)
 	{
@ -1790,23 +1882,24 @@ sub validate_ram
 	return(0);
 }

+### TODO: Make sure the appropriate SN ports are opened.
 # This stops (drbdadm down <server>) the storage for a given server on both nodes.
 sub manage_drbd_resource
 {
-	my ($anvil, $task) = @_;
+	my ($anvil, $task, $resource) = @_;
 	
-	read_server_definition($anvil);
-	validate_storage_drbd($anvil);
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		'server::drbd::peer::hostname' => $anvil->data->{server}{drbd}{peer}{hostname}, 
+	}});
+	
+	#read_server_definition($anvil);
+	#validate_storage_drbd($anvil);
 	
 	# Stop the resource on the peer, then stop it here.
-	my $server        = $anvil->data->{environment}{OCF_RESKEY_name};
 	my $peer_hostname = $anvil->data->{server}{drbd}{peer}{hostname};
-	my $peer_address  = $anvil->data->{server}{drbd}{peer}{address};
-	my $shell_call    = $anvil->data->{path}{exe}{drbdadm}." ".$task." ".$server;
+	my $shell_call    = $anvil->data->{path}{exe}{drbdadm}." ".$task." ".$resource;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-		server        => $server, 
 		peer_hostname => $peer_hostname,
-		peer_address  => $peer_address, 
 		shell_call    => $shell_call, 
 	}});
 	my ($output, $error, $return_code) = $anvil->Remote->call({
@ -1865,7 +1958,7 @@ sub read_server_definition
 	}
 	
 	# Still alive? Read it in.
-	my ($definition_xml) = read_file($anvil, $definition_file);
+	my $definition_xml = $anvil->Storage->read_file({file => $definition_file});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { definition_file => $definition_file }});
 	
 	my $xml = XML::Simple->new();
@ -1887,29 +1980,6 @@ sub read_server_definition
 	return(0);
 }

-# This reads in a file and returns the contents as a single string variable.
-sub read_file
-{
-	my ($anvil, $file) = @_;
-	
-	my $body       = "";
-	my $shell_call = $file; 
-	open (my $file_handle, "<".$shell_call) or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0015", variables => { shell_call => $shell_call, error => $! }});
-	while(<$file_handle>)
-	{
-		# This should not generate output.
-		chomp;
-		my $line =  $_;
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
-		
-		$body .= $line."\n";
-	}
-	close $file_handle;
-	
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { body => $body }});
-	return($body);
-}
-
 # This logs the details of this call.
 sub show_environment
 {
--- a/share/words.xml
+++ b/share/words.xml
@ -688,10 +688,10 @@ Output of: [#!variable!command!#] was;
 		<key name="log_0373">Checking that the DRBD device: [#!variable!device_path!#] is ready.</key>
 		<key name="log_0374">The server wants to use: [#!variable!device_path!#] as a hard drive, but we couldn't find the backing logical volume on this node.</key>
 		<key name="log_0375">The server wants to use: [#!variable!device_path!#] as a hard drive, but the backing logical volume: [#!variable!lv!#] doesn't exist on this node.</key>
-		<key name="log_0376">The server wants to use: [#!variable!device_path!#] as a hard drive, which is backed by the logical volume: [#!variable!lv!#]. Checking that these are ready.</key>
+		<key name="log_0376">The server wants to use: [#!variable!device_path!#] as a hard drive, which is backed by the logical volume: [#!variable!lv!#]. Will check that is is ready.</key>
 		<key name="log_0377">The attempt to read the DRBD status returned a non-zero code: [#!variable!return_code!#]. The returned output (if any) was: [#!variable!status_json!#].</key>
-		<key name="log_0378">DRBD is not loaded. Bringing it up now.</key>
-		<key name="log_0379">Bringing up the resource: [#!variable!resource!#] for the server's: [".#!variable!device_path!#."] disk.</key>
+		<key name="log_0378">The DRBD resource for this server is not running yet.</key>
+		<key name="log_0379">Bringing up the resource: [#!variable!resource!#] for the server's: [#!variable!device_path!#] disk.</key>
 		<key name="log_0380">The attempt to start the DRBD resource: [#!variable!resource!#] returned a non-zero code: [#!variable!return_code!#]. The returned output (if any) was: [#!variable!output!#].</key>
 		<key name="log_0381">Pausing briefly to give the resources time to start.</key>
 		<key name="log_0382">The attempt to read the DRBD status after bringing up the resource(s) for this server returned a non-zero code: [#!variable!return_code!#]. The returned output (if any) was: [#!variable!status_json!#].</key>
@ -701,8 +701,8 @@ Output of: [#!variable!command!#] was;
 		<key name="log_0386">The DRBD resource: [#!variable!resource!#] backing the device: [#!variable!device_path!#] was not able to start.</key>
 		<key name="log_0387">Checking that the peer's DRBD resources are Connected and UpToDate prior to migration.</key>
 		<key name="log_0388">The local replicated disk: [#!variable!device_path!#] is used by this server. Checking it out now.</key>
-		<key name="log_0389">The DRBD resource: [#!variable!resource!#] volume: [#!variable!volume!#] locat disk state is: [#!variable!disk_state!#]. Unsafe to boot the server unless the disk state is UpToDate.</key>
-		<key name="log_0390">The DRBD resource: [#!variable!resource!#] volume: [#!variable!volume!#] locat disk state is: [#!variable!disk_state!#], good.</key>
+		<key name="log_0389">The DRBD resource: [#!variable!resource!#] volume: [#!variable!volume!#] local disk state is: [#!variable!disk_state!#]. Unsafe to run the server unless the local disk state is UpToDate.</key>
+		<key name="log_0390">The DRBD resource: [#!variable!resource!#] volume: [#!variable!volume!#] local disk state is: [#!variable!disk_state!#], good.</key>
 		<key name="log_0391">Checking connection to: [#!variable!name!#].</key>
 		<key name="log_0392">The DRBD resource: [#!variable!resource!#] on the peer: [#!variable!name!#] is 'Primary'. Refusing to boot.</key>
 		<key name="log_0393">peer_short_name: [#!variable!peer_short_name!#], migration_target: [#!variable!migration_target!#].</key>
@ -719,6 +719,16 @@ Output of: [#!variable!command!#] was;
 		<key name="log_0404">The configured server name: [#!variable!name!#] needs: [#!variable!ram!# (#!variable!ram_bytes!# bytes)] of RAM, but only: #!variable!available_ram!# (#!variable!available_ram_bytes!# bytes)] are available!</key>
 		<key name="log_0405">The definition file: [#!variable!definition_file!#] for the server: [#!variable!server!#] does not exist here!</key>
 		<key name="log_0406">The definition file: [#!variable!definition_file!#] for the server: [#!variable!server!#] can not be read!</key>
+		<key name="log_0407">The server's disk: [#!variable!device_path!#] is part of the resource: [#!variable!resource!#] which was already started.</key>
+		<key name="log_0408">Taking down the resource: [#!variable!resource!#] for the server's: [#!variable!device_path!#] disk.</key>
+		<key name="log_0409">The server's disk: [#!variable!device_path!#] is part of the resource: [#!variable!resource!#] which was already taken down.</key>
+		<key name="log_0410">The DRBD resource: [#!variable!resource!#] local role is: [#!variable!role!#]. Promoting to primary now.</key>
+		<key name="log_0411">
+Failed to promote the DRBD resource: [#!variable!resource!#] primary. Expected a zero return code but got: [#!variable!return_code!#]. The output, if any, is below:
+====
+#!variable!output!#
+====
+		</key>
 		
 		<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
 		<key name="t_0000">Test</key>