Merge pull request #320 from ClusterLabs/anvil-tools-dev

Anvil tools dev

Anvil/Tools/Cluster.pm

@@ -502,7 +502,7 @@ ORDER BY
 				}
 			}
 			my $storage_group_member_uuid = $anvil->Database->insert_or_update_storage_group_members({
-				debug                                   => $debug, 
+				debug                                   => 2, 
 				storage_group_member_storage_group_uuid => $storage_group_uuid, 
 				storage_group_member_host_uuid          => $host_uuid, 
 				storage_group_member_vg_uuid            => $storage_group_member_vg_uuid, 

Anvil/Tools/Database.pm

@@ -6051,11 +6051,11 @@ WHERE
 					}});
 					
 					my $query = "DELETE FROM history.storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }});
 					$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 					
 					$query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
-					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }});
 					$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
 				}
 			}
@@ -13948,11 +13948,11 @@ SET
 WHERE  
     storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid)."
 ;";
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }});
 			$anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
     
 			$query = "DELETE FROM storage_group_members WHERE storage_group_member_uuid = ".$anvil->Database->quote($storage_group_member_uuid).";";
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { query => $query }});
 			$anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
 		}
 	}

Anvil/Tools/Email.pm

@@ -335,6 +335,7 @@ sub get_current_server
 	return($newest_mail_server_uuid);
 }
 
+
 =head2 get_next_server
 
 When two or more mail servers are configured, this will return the C<< mail_server_uuid >> of the mail server used in the most distant past. If two or more mail servers have never been used before, a random unused server is returned.
@@ -352,6 +353,11 @@ sub get_next_server
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Email->get_next_server()" }});
 	
+	if (not exists $anvil->data->{mail_servers}{mail_server})
+	{
+		$anvil->Database->get_mail_servers({debug => $debug});
+	}
+	
 	# If configured/running, the number of messages in queue is checked. If '0', 
 	# 'mail_server::queue_empty' is updated with the current time. If 1 or more, the time since the queue
 	# was last 0 is checked. If > 300, the mail server is reconfigured to use the mail server with the
@@ -360,11 +366,15 @@ sub get_next_server
 	my $oldest_mail_server_uuid = "";
 	foreach my $mail_server_uuid (keys %{$anvil->data->{mail_servers}{mail_server}})
 	{
-		my $last_used = $anvil->data->{mail_servers}{mail_server}{$mail_server_uuid}{last_used};
+		# HELO domain is 'DELETED' is the mail server is not used anymore
+		my $last_used   = $anvil->data->{mail_servers}{mail_server}{$mail_server_uuid}{last_used};
+		my $helo_domain = $anvil->data->{mail_servers}{mail_server}{$mail_server_uuid}{mail_server_helo_domain};
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 			mail_server_uuid => $mail_server_uuid,
 			last_used        => $last_used, 
+			helo_domain      => $helo_domain,
 		}});
+		next if $helo_domain eq "DELETED";
 		
 		if ($last_used < $oldest_mail_server_time)
 		{

Anvil/Tools/Get.pm

@@ -689,34 +689,103 @@ ORDER BY
 	}
 
 	# Check if the reserved RAM is overriden by the config
+	my $default_reserved = 8192;
+	if (not exists $anvil->data->{anvil_resources}{ram}{reserved})
+	{
+		$anvil->data->{anvil_resources}{ram}{reserved} = $default_reserved;
+	}
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		"anvil_resources::ram::reserved" => $anvil->data->{anvil_resources}{ram}{reserved}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{ram}{reserved}}).")",
+	}});
+	
+	$anvil->data->{anvil_resources}{ram}{reserved} =~ s/,//g;
+	$anvil->data->{anvil_resources}{ram}{reserved} =~ s/\s//g;
+	$anvil->data->{anvil_resources}{ram}{reserved} =~ s/MiB$//i;
+	$anvil->data->{anvil_resources}{ram}{reserved} =~ s/MB$//i;
+	$anvil->data->{anvil_resources}{ram}{reserved} =~ s/M$//i;
+	if ((not $anvil->data->{anvil_resources}{ram}{reserved}) or ($anvil->data->{anvil_resources}{ram}{reserved} =~ /\D/))
+	{
+		# Invalid value.
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0151", variables => { 
+			was => $anvil->data->{anvil_resources}{ram}{reserved}, 
+			set => $default_reserved,
+		}});
+		$anvil->data->{anvil_resources}{ram}{reserved} = $default_reserved;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			"anvil_resources::ram::reserved" => $anvil->data->{anvil_resources}{ram}{reserved},
+		}});
+	}
+	
+	#anvil::<anvil_uuid>::resources::ram::reserved
+	if (exists $anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved})
+	{
+		$anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved} =~ s/,//g;
+		$anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved} =~ s/\s//g;
+		$anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved} =~ s/MiB$//i;
+		$anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved} =~ s/MB$//i;
+		$anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved} =~ s/M$//i;
+		if ((not $anvil->data->{anvil_resources}{ram}{reserved}) or ($anvil->data->{anvil_resources}{ram}{reserved} =~ /\D/))
+		{
+			# Invalid value.
+			my $anvil_name = $anvil->Get->anvil_name_from_uuid({anvil_uuid => $anvil_uuid});
+			   $anvil_name = $anvil_uuid if not $anvil_name;
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0152", variables => { 
+				anvil => $anvil_name,
+				was   => $anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved}, 
+				set   => $anvil->data->{anvil_resources}{ram}{reserved},
+			}});
+			$anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved} = $anvil->data->{anvil_resources}{ram}{reserved};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				"anvil::${anvil_uuid}::resources::ram::reserved" => $anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved},
+			}});
+		}
+	
+		if ($anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved})
+		{
+			$anvil->data->{anvil_resources}{ram}{reserved} = $anvil->data->{anvil}{$anvil_uuid}{resources}{ram}{reserved};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				"anvil_resources::ram::reserved" => $anvil->data->{anvil_resources}{ram}{reserved},
+			}});
+		}
+	}
+	
 	my $ram_reserved = $anvil->Convert->human_readable_to_bytes({
 		base2 => 1,
-		size  => $anvil->data->{anvil_resources}{ram}{reserved}, 
+		size  => $anvil->data->{anvil_resources}{ram}{reserved}." MiB", 
 	});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		ram_reserved => $ram_reserved." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_reserved}).")",
+	}});
 	if (($ram_reserved eq "!!error!!") or 
-		(not $ram_reserved)            or 
+	    (not $ram_reserved)            or 
 	    ($ram_reserved < (2**30))      or 
 	    ($ram_reserved > $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{hardware}))
 	{
-		# The reserved RAM is invalid, so reset it.
+		# The reserved RAM is invalid, so reset it to 8 GiB
-		$ram_reserved = 0;
+		$ram_reserved = 8589934592;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			ram_reserved => $ram_reserved." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_reserved}).")",
+		}});
 	}
 
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-		"anvil_resources::ram::reserved" => $ram_reserved." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_reserved}).")",
-	}});
-
 	# Take 4 GiB or what was provided by the config off the available RAM for the host
-	$anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved}   = $ram_reserved ? $ram_reserved : (4*(2**30)); # Reserve 4 GiB by default or what's set in the config file.
+	$anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved}   = $ram_reserved;
 	$anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} -= $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved};
 	$anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} -= $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated};
-
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 		"anvil_resources::${anvil_uuid}::ram::allocated" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{allocated}}).")",
 		"anvil_resources::${anvil_uuid}::ram::reserved"  => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{reserved}}).")",
 		"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
 	}});
 	
+	if ($anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} < 0)
+	{
+		$anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available} = 0;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+			"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
+		}});
+	}
+	
 	# process bridges now
 	foreach my $bridge_name (sort {$a cmp $b} keys %{$anvil->data->{anvil_resources}{$anvil_uuid}{bridges}})
 	{

anvil.conf

@@ -193,6 +193,16 @@ striker::repo::extra-packages					=
 # Setting this to '0' will disable auto-management of the firewall.
 sys::manage::firewall						=	1
 
+# By default, Anvil! nodes reserve 8 GiB of RAM for host OS use. That is to say, if a node (pair) has 128GiB
+# of physical RAM, 120 GiB will be available for allocation to servers. Generally, this should NOT be reduced
+# as doing so could trigger oom-killer (out of memory killer) to terminate servers. This is an integer 
+# representing the reserved RAM in MiB
+# NOTE: Minimum is 1024, but really, never go below 4096 unless you _really_ know what you're doing.
+#anvil_resources::ram::reserved					=	8192
+
+# If you wish to alter the amount of reservered RAM for a single Anvil! node, you can use:
+#anvil::<anvil_uuid>::resources::ram::reserved			=	8192
+
 
 ### Server related options
 # This is the "short list" of servers shown when provisioning a new server. To see the full list of options,

anvil.spec.in

@@ -388,7 +388,7 @@ fi
 %{_usr}/lib/ocf/resource.d/alteeve/server
 
 %files dr
-#<placeholder for node specific files>
+#<placeholder for dr specific files>
 
 
 %changelog

notes

@@ -16,6 +16,8 @@ Common queries;
 * SELECT a.job_uuid, b.host_name, a.job_command, a.job_data, a.job_progress, a.job_status FROM jobs a, hosts b WHERE a.job_host_uuid = b.host_uuid AND a.job_progress != 100;
 * SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC;
 * SELECT a.dr_link_uuid, b.host_name, c.anvil_name, a.dr_link_note FROM dr_links a, hosts b, anvils c WHERE a.dr_link_host_uuid = b.host_uuid AND a.dr_link_anvil_uuid = c.anvil_uuid ORDER BY c.anvil_name ASC, b.host_name ASC;
+* SELECT a.storage_group_uuid, d.storage_group_member_uuid, b.anvil_name, a.storage_group_name, c.host_name, d.storage_group_member_vg_uuid, d.storage_group_member_note FROM storage_groups a, anvils b, hosts c, storage_group_members d WHERE a.storage_group_uuid = d.storage_group_member_storage_group_uuid AND a.storage_group_anvil_uuid = b.anvil_uuid AND c.host_uuid = d.storage_group_member_host_uuid ORDER BY a.storage_group_name ASC, c.host_name ASC;
+
 
 # Fail a resource for testing purposes.
 crm_resource --fail --resource srv02-b -N vm-a01n01

share/words.xml

@@ -386,6 +386,7 @@ The attempt to start the servers appears to have failed. The return code '0' was
 ====
 #!variable!output!#
 ====
+We're done waiting, exiting out.
 		</key>
 		<key name="error_0276"><![CDATA[No server specified to rename. Please use '--server <name>' or '--server-uuid <UUID>.]]></key>
 		<key name="error_0277">Could not find the server: [#!variable!server!#] on this Anvil! in the database.</key>
@@ -1531,6 +1532,7 @@ Note: This is a permanent action! If you protect this server again later, a full
  --install-media - File name or file UUID. Available discs are:]]></key>
 		<key name="job_0461"><![CDATA[                 > File name: [#!variable!name!#], file UUID: [#!variable!uuid!#], size: [#!variable!size!#]]]></key>
 		<key name="job_0462"><![CDATA[ --driver-disc   - (optional) A driver disc to be added as a second optical drive. Valid options are above.]]></key>
+		<key name="job_0463">Enabling the enable-safe-start daemon.</key>
 
 		<!-- Log entries -->
 		<key name="log_0001">Starting: [#!variable!program!#].</key>
@@ -3584,6 +3586,16 @@ The error was:
 		<key name="warning_0148">[ Warning ] - The IPMI stonith resource: [#!variable!resource!#] is in the role: [#!variable!role!#] (should be 'Started'). Will check the IPMI config now.</key>
 		<key name="warning_0149">[ Warning ] - Failed to find a valid IP address or password to be used to setup the DR host's IPMI.</key>
 		<key name="warning_0150">[ Warning ] - The test "fail file": [#!variable!fail_file!#] was found. So long as this file exists, the ocf:alteeve:server RA will return 'OCF_ERR_GENERIC' (exit code 1). Delete the file to resume normal operation.</key>
+		<key name="warning_0151">[ Warning ] - The configured reserved RAM was set to: [#!variable!was!#], which appears invalid. It must be an integer value representing the amount of RAM to reserve, in MiB. The reserved RAM is being set to: [#!variable!was!#].</key>
+		<key name="warning_0152">[ Warning ] - The configured reserved RAM was set to: [#!variable!was!#], which appears invalid. It must be an integer value representing the amount of RAM to reserve, in MiB. The reserved RAM is being set to: [#!variable!was!#].</key>
+		<key name="warning_0153">
+The attempt to start the servers appears to have failed. The return code '0' was expected, but: [#!variable!return_code!#] was received. The output was:
+====
+#!variable!output!#
+====
+We will wait: [#!variable!waiting!#] seconds and then try again. We'll give up if it keeps failing after: [#!variable!time_left!#] seconds.
+		</key>
+		
 	</language>
 	<!-- 日本語 -->
 	<language name="jp" long_name="日本語" description="Anvil! language file.">

tools/anvil-join-anvil

@@ -66,6 +66,15 @@ configure_pacemaker($anvil);
 # Configure DRBD
 configure_drbd($anvil);
 
+# Enable anvil-safe-start
+if (1)
+{
+	my ($return_code) = $anvil->System->enable_daemon({daemon => "anvil-safe-start.service"});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
+	update_progress($anvil, 99, "job_0094,!!daemon!anvil-safe-start.service!!");
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "anvil-safe-start.service" }});
+}
+
 update_progress($anvil, 100, "job_0128");
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"});
 

tools/anvil-manage-dr

@@ -226,6 +226,7 @@ sub handle_links
 			dr_link_note       => "user-created",
 		});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { returned_dr_link_uuid => $returned_dr_link_uuid }});
+		
 		print "\n".$anvil->Words->string({key => "log_0734", variables => {
 			host  => $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name},
 			anvil => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name},
@@ -533,7 +534,7 @@ sub sanity_check
 	# Get the Anvil! details.
 	$anvil->Database->get_hosts();
 	$anvil->Database->get_anvils();
-	$anvil->Database->get_storage_group_data();
+	$anvil->Database->get_storage_group_data({debug => 2});
 	$anvil->Database->get_dr_links({debug => 2});
 
 	# Does this Anvil! have at least one DR node? If there's only one, use it. If more than one, we need

tools/anvil-manage-files

@@ -554,6 +554,9 @@ sub check_incoming
 			full_path       => $full_path,
 		}});
 		
+		# Skip dot-files, they're usually files being uploaded.
+		next if $file_name =~ /^\./;
+		
 		# Do I know about this file? If so, is the file the same size? If either is no, calculate the md5sum.
 		my ($file_uuid, $recorded_size, $recorded_mtime, $recorded_md5sum) = get_file_db_info($anvil, "", $file_name);
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 

tools/anvil-manage-storage-groups

@@ -60,7 +60,7 @@ if (not $anvil->data->{sys}{database}{connections})
 
 $anvil->Database->get_hosts({include_deleted => 1});
 $anvil->Database->get_anvils();
-$anvil->Database->get_storage_group_data();
+$anvil->Database->get_storage_group_data({debug => 2});
 get_vg_data($anvil);
 get_storage_data($anvil);
 

tools/anvil-provision-server

@@ -2136,7 +2136,10 @@ sub interactive_ask_server_cpu
 	{
 		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
 		$anvil->Database->get_anvils();
-		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
+		$anvil->Get->available_resources({
+			debug      => 2,
+			anvil_uuid => $anvil_uuid,
+		});
 		
 		my $default_cpu = $anvil->data->{switches}{cpu};
 		if (not $default_cpu)
@@ -2201,7 +2204,10 @@ sub interactive_ask_server_ram
 	{
 		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
 		$anvil->Database->get_anvils();
-		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
+		$anvil->Get->available_resources({
+			debug      => 2,
+			anvil_uuid => $anvil_uuid,
+		});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			"anvil_resources::${anvil_uuid}::ram::available" => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvil_resources}{$anvil_uuid}{ram}{available}}).")",
 		}});
@@ -2299,7 +2305,10 @@ sub interactive_ask_server_storage_group
 	{
 		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
 		$anvil->Database->get_anvils();
-		$anvil->Get->available_resources({debug => 2, anvil_uuid => $anvil_uuid});
+		$anvil->Get->available_resources({
+			debug      => 2, 
+			anvil_uuid => $anvil_uuid,
+		});
 		
 		# I need a list of Storage groups,
 		my $say_ram               = $anvil->Convert->bytes_to_human_readable({"bytes" => $anvil->data->{new_server}{ram}});
@@ -2408,8 +2417,11 @@ sub interactive_ask_server_storage_size
 		}});
 		
 		$anvil->Database->get_anvils();
-		$anvil->Database->get_storage_group_data();
+		$anvil->Database->get_storage_group_data({debug => 2});
-		$anvil->Get->available_resources({debug => 2, anvil_uuid => $anvil_uuid});
+		$anvil->Get->available_resources({
+			debug      => 2, 
+			anvil_uuid => $anvil_uuid,
+		});
 		
 		$vg_size = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{vg_size};
 		$vg_free = $anvil->data->{anvil_resources}{$anvil_uuid}{storage_group}{$storage_group_uuid}{free_size};
@@ -2866,7 +2878,10 @@ sub interactive_ask_server_confirm
 		
 		my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
-		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid, debug => 2});
+		$anvil->Get->available_resources({
+			debug      => 2,
+			anvil_uuid => $anvil_uuid, 
+		});
 		
 		$anvil->data->{new_server}{name}          = "";
 		$anvil->data->{new_server}{uuid}          = "";

tools/anvil-report-usage

@@ -116,7 +116,10 @@ sub collect_anvil_data
 			}});
 		}
 		
-		$anvil->Get->available_resources({anvil_uuid => $anvil_uuid});
+		$anvil->Get->available_resources({
+			debug      => 2, 
+			anvil_uuid => $anvil_uuid,
+		});
 		my $cpu_cores         = $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{cores};
 		my $cpu_threads       = $anvil->data->{anvil_resources}{$anvil_uuid}{cpu}{threads};
 		my $say_cpu           = $anvil->Words->string({key => "message_0289", variables => {
@@ -338,7 +341,7 @@ sub show_anvils
 		{
 			$first_line .= " | ".sprintf("%-${longest_ram_used}s", $ram_used_string);
 		}
-		$first_line .= " | ".sprintf("%-${longest_ram_free}s", $ram_used_string);
+		$first_line .= " | ".sprintf("%-${longest_ram_free}s", $ram_free_string);
 		$first_line .= " | ".sprintf("%-${longest_bridge_string}s", $bridge_string);
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { first_line => $first_line }});
 		
@@ -625,22 +628,30 @@ sub collect_server_data
 	$anvil->data->{longest}{ip_address}     = 0;
 	foreach my $server_uuid (sort {$a cmp $b} keys %{$anvil->data->{servers}{server_uuid}})
 	{
-		my $server_name       = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
+		my $server_name  = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_name};
-		my $anvil_uuid        = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
+		my $anvil_uuid   = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid};
-		my $anvil_name        = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name};
+		my $anvil_name   = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_name};
-		my $server_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
+		my $server_ram   = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_ram_in_use};
-		my $server_ram        = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_ram_in_use};
+		my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
 		if ($anvil->data->{servers}{server_uuid}{$server_uuid}{server_configured_ram} > $server_ram)
 		{
 			$server_ram = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_configured_ram};
 		}
 		my $say_server_ram = $anvil->Convert->bytes_to_human_readable({'bytes' => $server_ram});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			's1:server_name' => $server_name,
+			's1:server_uuid'  => $server_uuid,
-			's2:anvil_name'  => $anvil_name, 
+			's2:server_name'  => $server_name,
-			's3:server_ram'  => $anvil->Convert->add_commas({number => $server_ram})." (".$say_server_ram.")", 
+			's3:anvil_uuid'   => $anvil_uuid, 
+			's4:anvil_name'   => $anvil_name, 
+			's5:server_state' => $server_state, 
+			's6:server_ram'   => $anvil->Convert->add_commas({number => $server_ram})." (".$say_server_ram.")", 
 		}});
 		
+		next if $server_state eq "DELETED";
+		
+		my $server_definition = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition => $server_definition }});
+		
 		my $target = $anvil->Get->short_host_name;
 		my $source = "from_db";
 		$anvil->Server->parse_definition({
@@ -722,6 +733,15 @@ sub collect_server_data
 		# have a matching node name.
 		my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
 		my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			's1:node1_host_uuid' => $node1_host_uuid, 
+			's2:node2_host_uuid' => $node2_host_uuid, 
+		}});
+		
+		if (not exists $anvil->data->{hosts}{host_uuid}{$node1_host_uuid})
+		{
+			$anvil->Database->get_hosts({debug => 2});
+		}
 		
 		# Get names.
 		my $node1_host_name       = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};

tools/anvil-safe-start

@@ -284,22 +284,55 @@ sub start_pacemaker
 		
 		### TODO: A lot more testing is needed for degraded single-node start later. 
 		###       Should we use --all, or wait for our peer? For now, we wait.
-		#my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start --all";
+		### NOTE: This can be racy during initial setup, calling the start before /etc/hosts is 
-		my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start";
+		###        populated. So this watches for that corner case.
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		my $wait_until = time + 120;
-		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+		my $waiting    = 1;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-			output      => $output,
+			wait_until => $wait_until,
-			return_code => $return_code, 
+			waiting    => $waiting, 
 		}});
-		if ($return_code)
+		while($waiting)
 		{
-			# What?! Fail out, we're done.
+			#my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start --all";
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0256", variables => {
+			my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 				output      => $output,
 				return_code => $return_code, 
 			}});
-			$anvil->nice_exit({exit_code => 1});
+			if ($return_code)
+			{
+				# Are we done waiting?
+				if (time > $wait_until)
+				{
+					# We're done.
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0256", variables => {
+						output      => $output,
+						return_code => $return_code, 
+					}});
+					$anvil->nice_exit({exit_code => 1});
+				}
+				else
+				{
+					# Report the error and sleep
+					my $time_left = $wait_until - time;
+					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0153", variables => {
+						output      => $output,
+						return_code => $return_code, 
+						time_left   => $time_left, 
+						waiting     => 10,
+					}});
+					sleep 10;
+				}
+			}
+			else
+			{
+				# Success!
+				$waiting = 0;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
+			}
 		}
 		
 		### TODO: We may implement the logic to fence our peer (similar to cman's post_join_delay' 
@@ -309,7 +342,7 @@ sub start_pacemaker
 		# the peer and, if the fence succeeds, unblock quorum.
 		my $start_time    = time;
 		my $wait_for_peer = $start_time + 120;
-		my $waiting       = 1;
+		   $waiting       = 1;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			start_time    => $start_time,
 			wait_for_peer => $wait_for_peer, 

tools/scancore

@@ -429,33 +429,17 @@ sub startup_tasks
 	});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { variable_uuid => $variable_uuid }});
 	
-	# If this is a node and we've been up for less than ten minutes, call anvil-safe-start as a 
+	# This used to call anvil-safe-start, which isn't done here anymore.
-	# background process. It will exit if it is disabled.
 	my $host_type = $anvil->Get->host_type();
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
 	if ($host_type eq "node")
 	{
-		my $uptime = $anvil->Get->uptime;
+		# For future use.
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { uptime => $uptime }});
-		if ($uptime < 600)
-		{
-			# Run it as a background task
-			my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-start'}.$anvil->Log->switches;
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0210", variables => { command => $shell_call }});
-			$anvil->System->call({shell_call => $shell_call, background => 1});
-		}
-		else
-		{
-			# Log that we've been up too long to auto-start the cluster.
-			my $say_uptime = $anvil->Convert->time({'time' => $uptime});
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0620", variables => { uptime => $say_uptime }});
-		}
 	}
 	elsif ($host_type eq "striker")
 	{
 		# We're a striker, so we're going to check for / remove transient database records on tables
 		# that always grow (temperature, power, etc) and whose data loses value as it ages.
-
 	}
 	
 	return(0);

units/Makefile.am

@@ -3,5 +3,6 @@ MAINTAINERCLEANFILES	= Makefile.in
 servicedir		= $(SYSTEMD_UNIT_DIR)
 dist_service_DATA	= \
 			  anvil-daemon.service \
+			  anvil-safe-start.service \
 			  scancore.service \
 			  striker-ui-api.service

units/anvil-safe-start.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Anvil! IA Platform - This service sanity checks and, if all is well, auto-starts the cluster and servers
+Wants=network.target
+
+[Service]
+ExecStart=/usr/sbin/anvil-safe-start
+RemainAfterExit=true
+Type=oneshot
+
+[Install]
+WantedBy=multi-user.target