anvil/tools/anvil-prep-database

#!/usr/bin/perl
# 
# This checks the state of the postgresql database server and, if necessary, configures it for external 
# access, initializes it and gets it running.
# 
# Exit codes;
# 0 = Normal exit.
# 1 = Failed to initialize postgres
# 2 = Failed to start postgres
# 3 = ScanCore user not set in the local ID in anvil.conf
# 4 = Failed to create the database user.
# 5 = PostgreSQL not installed.

use strict;
use warnings;
use Data::Dumper;
use Anvil::Tools;

my $THIS_FILE           =  ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory   =  ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
	$running_directory =~ s/^\./$ENV{PWD}/;
}

# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;

my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});

# Read switches
$anvil->Get->switches;

# Paths
$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});

my $local_uuid = $anvil->Database->get_local_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_uuid => $local_uuid }});

# If we didn't get the $local_uuid, then there is no entry for this system in anvil.conf yet, so we'll add it.
if (not $local_uuid)
{
	$local_uuid = add_to_local_config($anvil);
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_uuid => $local_uuid }});
	
	# If we still don't have a local_uuid, something went wrong.
	if (not $local_uuid)
	{
		print $anvil->Words->string({key => "error_0010"})."\n";
		$anvil->nice_exit({code => 1});
	}
}

# Now configure!
if ($local_uuid)
{
	# Start checks
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::service::postgresql" => $anvil->data->{sys}{service}{postgresql} }});
	
	my $running = $anvil->System->check_daemon({debug => 2, daemon => $anvil->data->{sys}{service}{postgresql}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { running => $running }});
	if ($running eq "2")
	{
		# Not installed.
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0144"});
		exit(5);
	}
	elsif (not $running)
	{
		# Do we need to initialize the databae?
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "path::configs::pg_hba.conf" => $anvil->data->{path}{configs}{'pg_hba.conf'} }});
		if (-e $anvil->data->{path}{configs}{'pg_hba.conf'})
		{
			# It already exists.
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "path::configs::pg_hba.conf" => $anvil->data->{path}{configs}{'pg_hba.conf'} }});
		}
		else
		{
			# Initialize.
			my $output = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output }});
			
			# Did it succeed?
			if (not -e $anvil->data->{path}{configs}{'pg_hba.conf'})
			{
				# Failed... 
				if ($output =~ /cannot create directory ‘(.*?)’: File exists/s)
				{
					my $file = $1;
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0139", variables => { file => $file }});
				}
				elsif ($output =~ /Initializing database ... failed, see (\/var\/.*?\.log)/s)
				{
					my $file = $1;
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0140", variables => { file => $file }});
				}
				else
				{
					$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0050"});
				}
				exit(1);
			}
			else
			{
				# Initialized!
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0055"});
			}
			
			# Setup postgresql.conf
			my $postgresql_backup = $anvil->data->{path}{directories}{backups}."/pgsql/postgresql.conf";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { postgresql_backup => $postgresql_backup }});
			$anvil->Storage->copy_file({
				source_file => $anvil->data->{path}{configs}{'postgresql.conf'}, 
				target_file => $postgresql_backup,
			});
			
			my $postgresql_conf     = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'postgresql.conf'}});
			my $update_file         = 1;
			my $new_postgresql_conf = "";
			foreach my $line (split/\n/, $postgresql_conf)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				if ($line =~ /^listen_addresses = '\*'/)
				{
					# No need to update.
					$update_file = 0;
					last;
				}
				elsif ($line =~ /^#listen_addresses = 'localhost'/)
				{
					# Inject the new listen_addresses
					$new_postgresql_conf .= "listen_addresses = '*'\n";
				}
				$new_postgresql_conf .= $line."\n";
			}
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_file => $update_file }});
			if ($update_file)
			{
				$anvil->Storage->write_file({
					file      => $anvil->data->{path}{configs}{'postgresql.conf'}, 
					body      => $new_postgresql_conf,
					user      => "postgres", 
					group     => "postgres",
					mode      => "0600",
					overwrite => 1,
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0056", variables => { file => $anvil->data->{path}{configs}{'postgresql.conf'} }});
			}
			
			# Setup pg_hba.conf now
			my $pg_hba_backup = $anvil->data->{path}{directories}{backups}."/pgsql/pg_hba.conf";
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pg_hba_backup => $pg_hba_backup }});
			
			$anvil->Storage->copy_file({
				source_file => $anvil->data->{path}{configs}{'pg_hba.conf'},
				target_file => $pg_hba_backup, 
			});
			my $pg_hba_conf     = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'pg_hba.conf'}});
			   $update_file     = 1;
			my $new_pg_hba_conf = "";
			foreach my $line (split/\n/, $pg_hba_conf)
			{
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
				if ($line =~ /^host\s+all\s+all\s+\all\s+md5$/)
				{
					# No need to update.
					$update_file = 0;
					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_file => $update_file }});
					last;
				}
				elsif ($line =~ /^# TYPE\s+DATABASE/)
				{
					# Inject the new listen_addresses
					$new_pg_hba_conf .= $line."\n";
					$new_pg_hba_conf .= "host\tall\t\tall\t\tall\t\t\tmd5\n";
				}
				else
				{
					$new_pg_hba_conf .= $line."\n";
				}
			}
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_file => $update_file }});
			if ($update_file)
			{
				$anvil->Storage->write_file({
					file      => $anvil->data->{path}{configs}{'pg_hba.conf'}, 
					body      => $new_pg_hba_conf,
					user      => "postgres", 
					group     => "postgres",
					mode      => "0600",
					overwrite => 1,
				});
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0057", variables => { file => $anvil->data->{path}{configs}{'pg_hba.conf'} }});
			}
		}
		
		# Start the daemon. '0' = started, anything else is a problem.
		my $return_code = $anvil->System->start_daemon({debug => 2, daemon => $anvil->data->{sys}{service}{postgresql}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
		if ($return_code eq "0")
		{
			# Started the daemon.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0059"});
			
			# Make sure it is enabled on boot.
			my $return_code = $anvil->System->enable_daemon({daemon => $anvil->data->{sys}{service}{postgresql}});
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
		}
		else
		{
			# Failed to start
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0094"});
			exit(2);
		}
	}
	
	# Create the .pgpass file, if needed.
	my $created_pgpass = 0;
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { 
		'path::secure::postgres_pgpass'     => $anvil->data->{path}{secure}{postgres_pgpass},
		"database::${local_uuid}::password" => $anvil->data->{database}{$local_uuid}{password}, 
	}});
	if ((not -e $anvil->data->{path}{secure}{postgres_pgpass}) && ($anvil->data->{database}{$local_uuid}{password}))
	{
		my $body = "*:*:*:postgres:".$anvil->data->{database}{$local_uuid}{password}."\n";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { body => $body }});
		$anvil->Storage->write_file({
			file      => $anvil->data->{path}{secure}{postgres_pgpass},  
			body      => $body,
			user      => "postgres", 
			group     => "postgres",
			mode      => "0600",
			overwrite => 1,
			secure    => 1,
		});
		if (-e $anvil->data->{path}{secure}{postgres_pgpass})
		{
			$created_pgpass = 1;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { created_pgpass => $created_pgpass }});
		}
	}
	
	# Does the database user exist?
	my $create_user   = 1;
	my $database_user = $anvil->data->{database}{$local_uuid}{user} ? $anvil->data->{database}{$local_uuid}{user} : $anvil->data->{sys}{database}{user};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_user => $database_user }});
	if (not $database_user)
	{
		# No database user defined
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0099", variables => { uuid => $local_uuid }});
		exit(3);
	}
	my $user_list = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { user_list => $user_list }});
	foreach my $line (split/\n/, $user_list)
	{
		if ($line =~ /^ $database_user\s+\|\s+(\d+)/)
		{
			# User exists already
			my $id = $1;
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0060", variables => { user => $database_user, id => $id }});
			$create_user = 0;
			last;
		}
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_user => $create_user }});
	if ($create_user)
	{
		# Create the user
		my $create_output = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole $database_user\"", source => $THIS_FILE, line => __LINE__});
		my $user_list     = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
		my $user_exists   = 0;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_output => $create_output, user_list => $user_list }});
		foreach my $line (split/\n/, $user_list)
		{
			if ($line =~ /^ $database_user\s+\|\s+(\d+)/)
			{
				# Success!
				my $id = $1;
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0095", variables => { user => $database_user, id => $id }});
				$user_exists = 1;
				last;
			}
		}
		if (not $user_exists)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0096", variables => { user => $database_user }});
			exit(4);
		}
		
		# Update/set the passwords.
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "database::${local_uuid}::password" => $anvil->data->{database}{$local_uuid}{password} }});
		if ($anvil->data->{database}{$local_uuid}{password})
		{
			foreach my $user ("postgres", $database_user)
			{
				my $update_output = $anvil->System->call({secure => 1, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c \\\"ALTER ROLE $user WITH PASSWORD '".$anvil->data->{database}{$local_uuid}{password}."';\\\"\"", source => $THIS_FILE, line => __LINE__});
				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { update_output => $update_output }});
				foreach my $line (split/\n/, $user_list)
				{
					if ($line =~ /ALTER ROLE/)
					{
						# Password set
						$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0100", variables => { user => $user }});
					}
				}
			}
		}
	}
	
	# Create the database, if needed.
	my $create_database = 1;
	my $database_name   = $anvil->data->{database}{$local_uuid}{name} ? $anvil->data->{database}{$local_uuid}{name} : $anvil->data->{sys}{database}{name};
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_name => $database_name }});
	
	my $database_list = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT datname FROM pg_catalog.pg_database;'\"", source => $THIS_FILE, line => __LINE__});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_list => $database_list }});
	foreach my $line (split/\n/, $database_list)
	{
		if ($line =~ /^ $database_name$/)
		{
			# Database already exists.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0105", variables => { database => $database_name }});
			$create_database = 0;
			last;
		}
	}
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_database => $create_database }});
	if ($create_database)
	{
		my $create_output = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createdb}."  --owner ".$database_user." ".$database_name."\"", source => $THIS_FILE, line => __LINE__});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_output => $create_output }});
		
		my $database_exists = 0;
		my $database_list   = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT datname FROM pg_catalog.pg_database;'\"", source => $THIS_FILE, line => __LINE__});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_list => $database_list }});
		foreach my $line (split/\n/, $database_list)
		{
			if ($line =~ /^ $database_name$/)
			{
				# Database created
				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0110", variables => { database => $database_name }});
				$database_exists = 1;
				last;
			}
		}
		if (not $database_exists)
		{
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0109", variables => { database => $database_name }});
			exit(5);
		}
	}
	
	# Remove the temporary password file.
	if (($created_pgpass) && (-e $anvil->data->{path}{secure}{postgres_pgpass}))
	{
		unlink $anvil->data->{path}{secure}{postgres_pgpass};
		if (-e $anvil->data->{path}{secure}{postgres_pgpass})
		{
			# Failed to unlink the file.
			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0107"});
		}
	}
	
	#####################################################################################################
	# NOTE: Below here is stuff that is for general setup. If it grows, we'll have to rename this tool. #
	#####################################################################################################
	
	### TODO: This will need to set the proper SELinux context.
	# Apache run scripts can't call the system UUID, so we'll write it to a text file.
	if (not -e $anvil->data->{path}{data}{host_uuid})
	{
		$anvil->Storage->write_file({
			file      => $anvil->data->{path}{data}{host_uuid}, 
			body      => $anvil->Get->host_uuid,
			user      => "apache", 
			group     => "apache",
			mode      => "0666",
			overwrite => 0,
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0011", variables => { file => $anvil->data->{path}{configs}{'postgresql.conf'} }});
	}
	
	# Log level 3 creates so much logging that it hits journald's rate limiting (1000 logs per 30 
	# seconds). So we need to disable it.
	if (not -e $anvil->data->{path}{configs}{'journald_anvil'})
	{
		# Write the file to disable journald rate limiting.
		my $body = "# This disables the rate limiting so that when log level is set to 3, log 
# entries aren't lost. If you want to override this, don't delete the file, 
# just comment out the lines below.

[Journal]
RateLimitInterval=0
RateLimitBurst=0
";
		$anvil->Storage->write_file({
			file      => $anvil->data->{path}{configs}{'journald_anvil'}, 
			body      => $body,
			user      => "root", 
			group     => "root",
			mode      => "0644",
			overwrite => 0,
		});
		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0012", variables => { file => $anvil->data->{path}{configs}{'journald_anvil'} }});
		
		my $shell_call = $anvil->data->{path}{exe}{systemctl}." restart systemd-journald.service";
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
		
		my $output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output }});
	}
}
else
{
	# Didn't find an entry for this machine. This is normal on nodes.
	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0143"});
}

$anvil->nice_exit({code => 0});


#############################################################################################################
# Functions                                                                                                 #
#############################################################################################################

# This adds this machine to the local anvil.conf file.
sub add_to_local_config
{
	my ($anvil) = @_;
	
	my $host_uuid       = $anvil->Get->host_uuid();
	my $local_uuid      = "";
	my $anvil_conf_body = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'anvil.conf'}});
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
		host_uuid       => $host_uuid,
		anvil_conf_body => $anvil_conf_body,
	}});
	
	# Setup the data we'll insert. The password will be changed after the user does the initial config.
	$anvil->data->{database}{$host_uuid}{host}     = "localhost";
	$anvil->data->{database}{$host_uuid}{port}     = "5432";
	$anvil->data->{database}{$host_uuid}{password} = "Initial1";
	$anvil->data->{database}{$host_uuid}{ping}     = "0";
	my $insert =  "database::".$host_uuid."::host		=	".$anvil->data->{database}{$host_uuid}{host}."\n";
	   $insert .= "database::".$host_uuid."::port		=	".$anvil->data->{database}{$host_uuid}{port}."\n";
	   $insert .= "database::".$host_uuid."::password	=	".$anvil->data->{database}{$host_uuid}{password}."\n";
	   $insert .= "database::".$host_uuid."::ping		=	".$anvil->data->{database}{$host_uuid}{ping}."\n\n";
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { insert => $insert }});
	   
	# Now inject the config.
	my $new_body    = "";
	my $config_seen = 0;
	my $test_line   = "database::${host_uuid}::";
	foreach my $line (split/\n/, $anvil_conf_body)
	{
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
		if ($line =~ /^$test_line/)
		{
			# It's already configured, abort.
			$config_seen = 1;
			$local_uuid  = $host_uuid;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
				config_seen => $config_seen,
				local_uuid  => $local_uuid, 
			}});
		}
		if ($line eq "### end db list ###")
		{
			$new_body .= $insert;
			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_body => $new_body }});
		}
		$new_body .= $line."\n";
	}
	
	# If we're here, we're ready to write it out.
	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { config_seen => $config_seen }});
	if (not $config_seen)
	{
		# Backup the original
		my $backup_file = $anvil->Storage->backup({file => $anvil->data->{path}{configs}{'anvil.conf'}});
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
		
		# Now update!
		$anvil->Storage->write_file({
			debug     => 2,
			file      => $anvil->data->{path}{configs}{'anvil.conf'}, 
			body      => $new_body, 
			user      => "admin", 
			group     => "admin", 
			mode      => "0644",
			overwrite => 1,
		});
		
		# Record the local UUID for returning to the caller.
		$local_uuid = $host_uuid;
		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_uuid => $local_uuid }});
	}
	
	return($local_uuid);
}