Merge pull request #216 from ClusterLabs/anvil-tools-dev

Updated scancore and anvil-daemon to check their RAM use at the end o…

Anvil/Tools/Database.pm

@@ -15691,6 +15691,16 @@ sub resync_databases
 		return(0);
 	}
 	
+	# If we're not a striker, don't resync ever.
+	my $host_type = $anvil->Get->host_type();
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_type => $host_type }});
+	if ($host_type ne "striker")
+	{
+		# Not a dashboard, don't resync
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0686"});
+		return(1);
+	}
+	
 	# If we're hosting servers, don't resync. Too high of a risk of oom-killer being triggered.
 	my $server_count = $anvil->Server->count_servers({debug => $debug});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { server_count => $server_count }});
@@ -15700,6 +15710,9 @@ sub resync_databases
 		return(0);
 	}
 	
+	# Before resync, age out the data in each DB
+	$anvil->Database->_age_out_data({debug => $debug});
+	
 	### NOTE: Don't sort this array, we need to resync in the order that the user passed the tables to us
 	###       to avoid trouble with primary/foreign keys.
 	# We're going to use the array of tables assembles by _find_behind_databases() stored in 
@@ -16726,7 +16739,7 @@ sub _age_out_data
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { table => $table }});
 		
 		# Does the table exist?
-		$query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename='scan_apc_pdus' AND schemaname='public';";
+		$query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->Database->quote($table)." AND schemaname='public';";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
 		
 		my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];

Anvil/Tools/ScanCore.pm

@@ -199,7 +199,7 @@ sub agent_startup
 	if (($anvil->data->{scancore}{$agent}{disable}) && (not $anvil->data->{switches}{force}))
 	{
 		# Exit.
-		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0646", variables => { program => $THIS_FILE }});
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0646", variables => { program => $agent }});
 		$anvil->nice_exit({exit_code => 0});
 	}
 	

Anvil/Tools/System.pm

@@ -654,6 +654,159 @@ sub check_memory
 }
 
 
+=head2 check_ram_use
+
+This is meant to be used by daemons to check how much RAM it is using. It returns an anonymous array with the first value being C<< 0 >> if the in-use RAM is below the maximum, and C<< 1 >> it the in-use RAM is too high. The second value is the amount of RAM in use, in bytes. If the program is not found to be running, C<< 2, 0 >> is returned.
+
+ my ($problem, $used_ram) = $anvil->System->check_ram_use({
+ 	program => $THIS_FILE,
+ 	max_ram => 1073741824,
+ });
+
+Parameters;
+
+=head3 program (required)
+
+This is generally C<< $THIS_FILE >>. Though this could be used to check the RAM use of other programs.
+
+=head3 max_ram (optional, default '1073741824' (1 GiB))
+
+This is the limit allowed. If the in-use RAM is greater than this amount, an alert will be generated and sent. 
+
+=cut
+sub check_ram_use
+{
+	my $self      = shift;
+	my $parameter = shift;
+	my $anvil     = $self->parent;
+	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->check_ram_use()" }});
+	
+	my $program = defined $parameter->{program} ? $parameter->{program} : "";
+	my $max_ram = defined $parameter->{max_ram} ? $parameter->{max_ram} : 1073741824;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		program => $program, 
+		max_ram => $max_ram, 
+	}});
+	
+	# Find the PID(s) of the program.
+	my $problem  = 0;
+	my $ram_used = 0;
+	
+	# See if we're a daemon running under systemctl. If so, the memory reported includes all spawned 
+	# child programs, swap, etc. Much more thorough.
+	my $shell_call = $anvil->data->{path}{exe}{systemctl}." status ".$program." --lines=0";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
+	
+	my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		output      => $output, 
+		return_code => $return_code,
+	}});
+	foreach my $line (split/\n/, $output)
+	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
+		if ($line =~ /Memory: (.*)?/)
+		{
+			my $memory   = $1;
+			my $in_bytes = $anvil->Convert->human_readable_to_bytes({size => $memory, base2 => 1});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				memory   => $memory,
+				in_bytes => $anvil->Convert->add_commas({number => $in_bytes})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $in_bytes}).")",
+			}});
+			if ($in_bytes =~ /^\d+$/)
+			{
+				$ram_used = $in_bytes;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					ram_used => $anvil->Convert->add_commas({number => $ram_used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}).")",
+				}});
+			}
+			last;
+		}
+	}
+	
+	# If we didn't get the RAM from systemctl, read smaps
+	if (not $ram_used)
+	{
+		my $pids = $anvil->System->pids({debug => $debug, program_name => $program});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { pids => $pids }});
+
+		my $pids_found = @{$pids};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { pids_found => $pids_found }});
+
+		if (not $pids_found)
+		{
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "warning_0135", variables => { program => $program }});
+			return(2, 0);
+		}
+
+		# Read in the smaps for each pid
+		foreach my $pid (sort {$a cmp $b} @{$pids})
+		{
+			my $smaps_path = "/proc/".$pid."/smaps";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { smaps_path => $smaps_path }});
+			
+			# This will store the amount of RAM used by this specific PID.
+			$anvil->data->{memory}{pid}{$pid} = 0;
+			
+			if (not -e $smaps_path)
+			{
+				# It is possible that the program just closed.
+				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0433", variables => { pid => $pid }});
+				next;
+			}
+			
+			# Read in the file.
+			my $body = $anvil->Storage->read_file({debug => $debug, file => $smaps_path});
+			foreach my $line (split/\n/, $body)
+			{
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
+				if ($line =~ /^Private_Dirty:\s+(\d+) (.*B)$/)
+				{
+					my $size = $1;
+					my $type = $2;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						type => $type,
+						size => $size,
+					}});
+					next if not $size;
+					next if $size =~ /\D/;
+					
+					# This uses 'kB' for 'KiB' >_>
+					$type =  lc($type);
+					$type =~ s/b$/ib/ if $type !~ /ib$/;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { type => $type }});
+					
+					my $size_in_bytes = $anvil->Convert->human_readable_to_bytes({size => $size, type => $type, base2 => 1});
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						size_in_bytes => $anvil->Convert->add_commas({number => $size_in_bytes})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size_in_bytes}).")",
+					}});
+					
+					$anvil->data->{memory}{pid}{$pid} += $size_in_bytes;
+					$ram_used                         += $size_in_bytes;
+					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+						"memory::pid::${pid}" => $anvil->Convert->add_commas({number => $anvil->data->{memory}{pid}{$pid}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{memory}{pid}{$pid}}).")",
+						ram_used              => $anvil->Convert->add_commas({number => $ram_used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}).")",
+					}});
+				}
+			}
+		}
+	}
+	
+	# Are we using too much RAM?
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+		max_ram  => $anvil->Convert->add_commas({number => $max_ram})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $max_ram}).")",
+		ram_used => $anvil->Convert->add_commas({number => $ram_used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}).")",
+	}});
+	if ($ram_used > $max_ram)
+	{
+		$problem = 1;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { problem => $problem }});
+	}
+	
+	return($problem, $ram_used);
+}
+
 =head2 check_ssh_keys
 
 This method does several things;

cgi-bin/striker

@@ -6451,6 +6451,7 @@ sub process_power
 		my $job_description = "job_0006";
 		my $say_title       = "#!string!job_0005!#";
 		my $say_description = "#!string!job_0006!#";
+		my $say_reason      = "log_0199";
 		if ($task eq "poweroff")
 		{
 			$job_command     = $anvil->data->{path}{exe}{'anvil-manage-power'}." --poweroff -y".$anvil->Log->switches;
@@ -6458,7 +6459,9 @@ sub process_power
 			$job_description = "job_0008";
 			$say_title       = "#!string!job_0007!#";
 			$say_description = "#!string!job_0008!#";
+			$say_reason      = "log_0200";
 		}
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => $say_reason }});
 		my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
 			file            => $THIS_FILE, 
 			line            => __LINE__, 

notes

@@ -771,10 +771,10 @@ mediawiki on EL8 install notes (starting from a minimal install);
 dnf module reset php
 dnf module enable php:7.4
 
-# PgSQL
+# All
 dnf install httpd php php-gd php-xml php-mbstring php-json \
             vim bash-completion wget tar rsync mlocate php-pecl-apcu \
-            memcached php-pear icu php-intl php-pgsql bzip2 
+            memcached php-pear icu php-intl php-pgsql bzip2 mod_ssl
 
 ### PostgreSQL
 dnf install postgresql-server postgresql-plperl 
@@ -783,7 +783,9 @@ systemctl start postgresql.service
 systemctl enable postgresql.service 
 
 ### MariaDB
-dnf install php-mysqlnd php-gd php-xml mariadb-server mariadb
+dnf install httpd php php-gd php-xml php-mbstring php-json \
+            php-mysqlnd php-gd php-xml mariadb-server mariadb \
+
 systemctl start mariadb
 mysql_secure_installation
 |Set root password? [Y/n] y
@@ -796,25 +798,25 @@ mysql_secure_installation
 |Reload privilege tables now? [Y/n] y
 mysql -u root -p
 ### In mariadb
-MariaDB [(none)]> CREATE DATABASE digimer_wiki;
-MariaDB [(none)]> CREATE USER 'digimer'@'localhost' IDENTIFIED BY 'Initial1';
-MariaDB [(none)]> GRANT ALL PRIVILEGES ON digimer_wiki.* TO 'digimer'@'localhost';
+MariaDB [(none)]> CREATE DATABASE an_wiki;
+MariaDB [(none)]> CREATE USER 'alteeve'@'localhost' IDENTIFIED BY 'experience tell mineral';
+MariaDB [(none)]> GRANT ALL PRIVILEGES ON an_wiki.* TO 'alteeve'@'localhost';
 MariaDB [(none)]> FLUSH PRIVILEGES;
 MariaDB [(none)]> SHOW DATABASES;
 +--------------------+
 | Database           |
 +--------------------+
-| digimer_wiki       |
+| an_wiki       |
 | information_schema |
 | mysql              |
 | performance_schema |
 +--------------------+
-MariaDB [(none)]> SHOW GRANTS FOR 'digimer'@'localhost';
+MariaDB [(none)]> SHOW GRANTS FOR 'alteeve'@'localhost';
 +----------------------------------------------------------------------------------------------------------------+
 | Grants for digimer@localhost                                                                                   |
 +----------------------------------------------------------------------------------------------------------------+
 | GRANT USAGE ON *.* TO `digimer`@`localhost` IDENTIFIED BY PASSWORD '*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' |
-| GRANT ALL PRIVILEGES ON `digimer_wiki`.* TO `digimer`@`localhost`                                              |
+| GRANT ALL PRIVILEGES ON `an_wiki`.* TO `digimer`@`localhost`                                              |
 +----------------------------------------------------------------------------------------------------------------+
 MariaDB [(none)]> exit
 # Back to terminal
@@ -863,6 +865,38 @@ tar -xvzf mediawiki-1.37.1.tar.gz
 cd /var/www/html
 ln -s ../mediawiki-1.37.1 ./w
 
+systemctl enable httpd.service
+systemctl enable memcached.service 
+systemctl start httpd.service 
+systemctl start memcached.service
+
+firewall-cmd --zone=public --add-service=http --permanent 
+firewall-cmd --zone=public --add-service=https --permanent 
+firewall-cmd --reload 
+
+### Certbot / Let's Encrypt
+# EPEL / snapd
+dnf config-manager --set-enabled powertools
+dnf install epel-release epel-next-release
+dnf install snapd
+systemctl enable --now snapd.socket
+ln -s /var/lib/snapd/snap /snap
+
+### Setup vhost
+# httpd.conf
+
+
+### Log out and back in to ensure snapd path
+# If the next step fails with "too early for operation, device not yet seeded or device model not acknowledged", restart snapd
+snap install core 
+snap refresh core
+snap install --classic certbot
+
+# certbot
+certbot --apache
+
+# answer questions
+
 ====
 
 Dell S4128T-ON Configuration
@@ -1257,6 +1291,33 @@ rs-striker03(config-if)#switchport access vlan 100
 rs-striker03(config-if)#no shutdown 
 rs-striker03(config-if)#exit
 
+#### NOTE: Put IP on VID 1! 
+
+rs-switch03(config)#show vlan
+
+VLAN   Name                             Ports          Type
+-----  ---------------                  -------------  --------------
+1      default                          Po1-128,       Default
+                                        Gi1/0/1-12,
+                                        Te1/0/1-4,
+                                        Gi2/0/1-12,
+                                        Te2/0/1-4
+300    IFN1                             Gi1/0/13-24,   Static
+                                        Gi2/0/13-24
+
+rs-switch03(config)#interface vlan 1
+
+rs-switch03(config-if-vlan1)#ip address 10.201.1.3 255.255.0.0
+
+rs-switch03(config-if-vlan1)#exit     
+
+rs-switch03(config)#exit
+
+rs-switch03#copy running-config startup-config 
+
+###########################
+
+
 
 rs-striker03#show vlan 
 

scancore-agents/scan-network/scan-network

@@ -888,8 +888,8 @@ ORDER BY
 					my $queries = [];
 					push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
-					push @{$queries}, "DELETE FROM history.bondss WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
-					push @{$queries}, "DELETE FROM bondss WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
+					push @{$queries}, "DELETE FROM history.bonds WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
+					push @{$queries}, "DELETE FROM bonds WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					push @{$queries}, "DELETE FROM history.bridges WHERE bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					push @{$queries}, "DELETE FROM bridges WHERE bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					
@@ -920,8 +920,8 @@ ORDER BY
 					my $queries = [];
 					push @{$queries}, "DELETE FROM history.network_interfaces WHERE network_interface_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					push @{$queries}, "DELETE FROM network_interfaces WHERE network_interface_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
-					push @{$queries}, "DELETE FROM history.bondss WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
-					push @{$queries}, "DELETE FROM bondss WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
+					push @{$queries}, "DELETE FROM history.bonds WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
+					push @{$queries}, "DELETE FROM bonds WHERE bond_bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					push @{$queries}, "DELETE FROM history.bridges WHERE bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					push @{$queries}, "DELETE FROM bridges WHERE bridge_uuid = ".$anvil->Database->quote($bridge_uuid).";";
 					

share/words.xml

@@ -502,6 +502,7 @@ The output, if any, was;
 		</key>
 		<key name="error_0355">Failed to load the database file: [#!variable!file!#]. Deleting it so it's not considered in the next load attempt.</key>
 		<key name="error_0356">Failed to read the kernel release on the host: [#!variable!target!#]. The return code was: [#!variable!return_code!#] (expected '0') and the release output, if any, was: [#!variable!output!#].</key>
+		<key name="error_0357">The program: [#!variable!program!#] is using: [#!variable!ram_used!#] (#!variable!ram_used_bytes!# Bytes). This is probably caused by a memory leak, so we will now exit so that systemctl can restart us. If this is happening repeatedly, please contact support.</key>
 		
 		<!-- Files templates -->
 		<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->
@@ -1522,8 +1523,8 @@ The database connection error was:
 		<key name="log_0196">Failed to reconnect to the database, and now no connections remail. Exiting.</key>
 		<key name="log_0197"><![CDATA[System->maintenance_mode() was passed an invalid 'set' value: [#!variable!set!#]. No action taken.]]></key>
 		<key name="log_0198">The user: [#!variable!user!#] logged out successfully.</key>
-		<key name="log_0199">A system reboot is required, setting the database flag.</key>
-		<key name="log_0200">A system reboot is required, setting the database flag.</key>
+		<key name="log_0199">A system reboot has been requested via the Striker UI.</key>
+		<key name="log_0200">A system power-off has been requested via the Striker UI.</key>
 		<key name="log_0201">Unable to connect to any database. Will try to initialize the local system and then try again.</key>
 		<key name="log_0202">Failed to connect to any databases. Skipping the loop of the daemon.</key>
 		<key name="log_0203">Disconnected from all databases. Will reconnect when entering the main loop.</key>
@@ -2077,6 +2078,13 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
 		<key name="log_0683">Enabling 'ping' for all users.</key>
 		<key name="log_0684">The network interface: [#!variable!nic!#] on the host: [#!variable!host!#] is recorded in the 'history.network_interfaces' table, but has not corresponding entry in the public table. Removing it.</key>
 		<key name="log_0685">[ Note ] - The network bridge: [#!variable!name!#] with 'bridge_uuid': [#!variable!uuid!#] is a duplicate, removing it from the database(s).</key>
+		<key name="log_0686">Skipping resync, not a Striker dashboard.</key>
+		<key name="log_0687">### REBOOT REQUESTED ### - [#!variable!reason!#]</key>
+		<key name="log_0688">Reboot flag set by command line switch to 'anvil-manage-power'.</key>
+		<key name="log_0689">Poweroff flag set by command line switch to 'anvil-manage-power'.</key>
+		<key name="log_0690">Kernel updated, reboot queued.</key>
+		<key name="log_0691">Requested to power-off as part of the anvil-safe-stop job.</key>
+		<key name="log_0692">The anvil-safe-stop job has completed and will now power off.</key>
 		
 		<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
 		<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>
@@ -3107,6 +3115,7 @@ We will sleep a bit and try again.
 		<key name="warning_0132">[ Warning ] - Failed to build or install the DRBD kernel module! It is very unlikely that this machine will be able to run any servers until this is fixed.</key>
 		<key name="warning_0133">[ Warning ] - Table: [history.#!variable!table!#] not found.</key>
 		<key name="warning_0134">[ Warning ] - Holding off starting the cluster. Tested access to ourself, and failed. Is '/etc/hosts' populated? Will try again in ten seconds.</key>
+		<key name="warning_0135">[ Warning ] - The program: [#!variable!program!#] was not found to be running.</key>
 
 		<!-- The entries below here are not sequential, but use a key to find the entry. -->
 		<!-- Run 'striker-parse-os-list to find new entries. --> 

tools/anvil-daemon

@@ -246,6 +246,9 @@ while(1)
 		$anvil->nice_exit({exit_code => 0});
 	}
 	
+	# Check how much RAM we're using.
+	check_ram($anvil);
+	
 	# Disconnect from the database(s) and sleep now.
 	$anvil->Database->disconnect();
 	sleep(2);
@@ -258,6 +261,41 @@ $anvil->nice_exit({exit_code => 0});
 # Functions                                                                                                 #
 #############################################################################################################
 
+# If we're using too much ram, send an alert and exit.
+sub check_ram
+{
+	my ($anvil) = @_;
+	
+	# Problem 0 == ok, 1 == too much ram used, 2 == no pid found
+	my ($problem, $ram_used) = $anvil->System->check_ram_use({program => $THIS_FILE});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		problem  => $problem, 
+		ram_used => $anvil->Convert->add_commas({number => $ram_used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}).")", 
+	}});
+	if ($problem)
+	{
+		# Send an alert and exit.
+		$anvil->Alert->register({alert_level => "notice", message => "error_0357", variables => { 
+			program        => $THIS_FILE,
+			ram_used       => $anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}),
+			ram_used_bytes => $anvil->Convert->add_commas({number => $ram_used}),
+		}, set_by => $THIS_FILE, sort_position => 0});
+		$anvil->Email->send_alerts();
+		
+		# Log the same
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0357", variables => { 
+			program        => $THIS_FILE,
+			ram_used       => $anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}),
+			ram_used_bytes => $anvil->Convert->add_commas({number => $ram_used}),
+		}});
+		
+		# Exit with RC0 so that systemctl restarts 
+		$anvil->nice_exit({exit_code => 0});
+	}
+	
+	return(0);
+}
+
 # Check to see if we're mapping the network on this host.
 sub check_if_mapping
 {
@@ -571,6 +609,9 @@ sub handle_periodic_tasks
 					{
 						$first_uuid = $uuid; 
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { first_uuid => $first_uuid }});
+						
+						# Skip the first UUID so it doesn't evaluate for shutdown.
+						next;
 					}
 					elsif ($uuid eq $host_uuid)
 					{
@@ -1291,7 +1332,7 @@ sub prep_database
 	{
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			prep_database                => $prep_database,
-			"sys}{database}{connections" => $anvil->data->{sys}{database}{connections}, 
+			"sys::database::connections" => $anvil->data->{sys}{database}{connections}, 
 		}});
 		if ($prep_database)
 		{

tools/anvil-manage-power

@@ -130,6 +130,7 @@ if ($anvil->data->{switches}{'reboot-needed'} eq "1")
 	# Enable
 	if (not $reboot_needed)
 	{
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => "log_0688" }});
 		$reboot_needed = $anvil->System->reboot_needed({debug => 2, set => 1});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
 		print $anvil->Words->string({key => "message_0048"})."\n";
@@ -246,6 +247,8 @@ sub do_poweroff
 	
 	# Make sure the 'reboot needed' flag is set. When 'anvil-daemon' starts, it will use this to confirm
 	# that it is starting post-reboot and clear it.
+	my $say_reason = $task eq "poweroff" ? "log_0689" : "log_0688";
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => $say_reason }});
 	$reboot_needed = $anvil->System->reboot_needed({debug => 2, set => 1});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
 	

tools/anvil-safe-stop

@@ -101,6 +101,7 @@ if ($anvil->data->{switches}{'job-uuid'})
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 				'switches::power-off' => $anvil->data->{switches}{'power-off'},
 			}});
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => "log_0691" }});
 		}
 		if ($line =~ /stop-reason=(.*?)$/)
 		{
@@ -156,6 +157,7 @@ if ($anvil->data->{switches}{'power-off'})
 		host_status => "stopping",
 	});
 	
+	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => "log_0692" }});
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0325"});
 	$anvil->Job->update_progress({progress => 100, message => "job_0325"});
 	

tools/anvil-update-system

@@ -173,8 +173,7 @@ sub run_os_update
 		if ($line =~ /^kernel /)
 		{
 			# Reboot will be needed.
-			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0199"});
-			
+			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0687", variables => { reason => "log_0690" }});
 			my $reboot_needed = $anvil->System->reboot_needed({set => 1});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { reboot_needed => $reboot_needed }});
 		}

tools/scancore

@@ -163,6 +163,9 @@ while(1)
 	# Clean up
 	cleanup_after_run($anvil);
 	
+	# Check how much RAM we're using.
+	check_ram($anvil);
+	
 	# Sleep until it's time to run again.
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0249", variables => { 
 		run_interval => $run_interval,
@@ -181,6 +184,41 @@ $anvil->nice_exit({exit_code => 0});
 # Functions                                                                                                 #
 #############################################################################################################
 
+# If we're using too much ram, send an alert and exit.
+sub check_ram
+{
+	my ($anvil) = @_;
+	
+	# Problem 0 == ok, 1 == too much ram used, 2 == no pid found
+	my ($problem, $ram_used) = $anvil->System->check_ram_use({program => $THIS_FILE});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		problem  => $problem, 
+		ram_used => $anvil->Convert->add_commas({number => $ram_used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}).")", 
+	}});
+	if ($problem)
+	{
+		# Send an alert and exit.
+		$anvil->Alert->register({alert_level => "notice", message => "error_0357", variables => { 
+			program        => $THIS_FILE,
+			ram_used       => $anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}),
+			ram_used_bytes => $anvil->Convert->add_commas({number => $ram_used}),
+		}, set_by => $THIS_FILE, sort_position => 0});
+		$anvil->Email->send_alerts();
+		
+		# Log the same
+		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0357", variables => { 
+			program        => $THIS_FILE,
+			ram_used       => $anvil->Convert->bytes_to_human_readable({'bytes' => $ram_used}),
+			ram_used_bytes => $anvil->Convert->add_commas({number => $ram_used}),
+		}});
+		
+		# Exit with RC0 so that systemctl restarts 
+		$anvil->nice_exit({exit_code => 0});
+	}
+	
+	return(0);
+}
+
 # This cleans things up after a scan run has completed.
 sub cleanup_after_run
 {

tools/striker-auto-initialize-all

@@ -1425,7 +1425,7 @@ sub striker_stage1
 {
 	my ($anvil) = @_;
 	
-	### TODO: Validate all steps up front before starting anything.
+	# Validate
 	if ((not defined $anvil->data->{base}{organization_name}) or (not $anvil->data->{base}{organization_name}))
 	{
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0243", variables => { variable => 'base::organization_name' }});