anvil/tools/striker-prep-database
Digimer 1a520b03d5 * Cleaned up a lot of logging in anvil-daemon and tools it calls.
* Deleted anvil-jobs as it never ended up being used.

Signed-off-by: Digimer <digimer@alteeve.ca>
2021-02-08 13:39:34 -05:00

460 lines
20 KiB
Perl
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/perl
#
# This checks the state of the postgresql database server and, if necessary, configures it for external
# access, initializes it and gets it running.
#
# Exit codes;
# 0 = Normal exit.
# 1 = Failed to initialize postgres
# 2 = Failed to start postgres
# 3 = ScanCore user not set in the local ID in anvil.conf
# 4 = Failed to create the database user.
# 5 = PostgreSQL not installed.
use strict;
use warnings;
use Data::Dumper;
use Anvil::Tools;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
$anvil->Get->switches;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
my $local_uuid = $anvil->Database->get_local_uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { local_uuid => $local_uuid }});
# If we didn't get the $local_uuid, then there is no entry for this system in anvil.conf yet, so we'll add it.
if (not $local_uuid)
{
$local_uuid = add_to_local_config($anvil);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { local_uuid => $local_uuid }});
}
# Now configure!
if ($local_uuid)
{
# Start checks
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "sys::service::postgresql" => $anvil->data->{sys}{daemon}{postgresql} }});
my $running = $anvil->System->check_daemon({debug => 3, daemon => $anvil->data->{sys}{daemon}{postgresql}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { running => $running }});
if ($running eq "2")
{
# Not installed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0144"});
$anvil->nice_exit({exit_code => 5});
}
elsif (not $running)
{
# Do we need to initialize the databae?
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "path::configs::pg_hba.conf" => $anvil->data->{path}{configs}{'pg_hba.conf'} }});
if (-e $anvil->data->{path}{configs}{'pg_hba.conf'})
{
# It already exists.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "path::configs::pg_hba.conf" => $anvil->data->{path}{configs}{'pg_hba.conf'} }});
}
else
{
# Initialize.
my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
# Did it succeed?
if (not -e $anvil->data->{path}{configs}{'pg_hba.conf'})
{
# Failed...
if ($output =~ /cannot create directory (.*?): File exists/s)
{
my $file = $1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0139", variables => { file => $file }});
}
elsif ($output =~ /Initializing database ... failed, see (\/var\/.*?\.log)/s)
{
my $file = $1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0140", variables => { file => $file }});
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0050"});
}
$anvil->nice_exit({exit_code => 1});
}
else
{
# Initialized!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0055"});
}
# Setup postgresql.conf
my $postgresql_backup = $anvil->data->{path}{directories}{backups}."/pgsql/postgresql.conf";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { postgresql_backup => $postgresql_backup }});
$anvil->Storage->copy_file({
source_file => $anvil->data->{path}{configs}{'postgresql.conf'},
target_file => $postgresql_backup,
});
my $postgresql_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'postgresql.conf'}});
my $update_file = 1;
my $new_postgresql_conf = "";
foreach my $line (split/\n/, $postgresql_conf)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /^listen_addresses = '\*'/)
{
# No need to update.
$update_file = 0;
last;
}
elsif ($line =~ /^#listen_addresses = 'localhost'/)
{
# Inject the new listen_addresses
$new_postgresql_conf .= "listen_addresses = '*'\n";
}
$new_postgresql_conf .= $line."\n";
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_file => $update_file }});
if ($update_file)
{
$anvil->Storage->write_file({
debug => 2,
file => $anvil->data->{path}{configs}{'postgresql.conf'},
body => $new_postgresql_conf,
user => "postgres",
group => "postgres",
mode => "0600",
overwrite => 1,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0056", variables => { file => $anvil->data->{path}{configs}{'postgresql.conf'} }});
}
# Setup pg_hba.conf now
my $pg_hba_backup = $anvil->data->{path}{directories}{backups}."/pgsql/pg_hba.conf";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pg_hba_backup => $pg_hba_backup }});
$anvil->Storage->copy_file({
source_file => $anvil->data->{path}{configs}{'pg_hba.conf'},
target_file => $pg_hba_backup,
});
my $pg_hba_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'pg_hba.conf'}});
$update_file = 1;
my $new_pg_hba_conf = "";
foreach my $line (split/\n/, $pg_hba_conf)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /^host\s+all\s+all\s+\all\s+md5$/)
{
# No need to update.
$update_file = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_file => $update_file }});
last;
}
elsif ($line =~ /^# TYPE\s+DATABASE/)
{
# Inject the new listen_addresses
$new_pg_hba_conf .= $line."\n";
$new_pg_hba_conf .= "host\tall\t\tall\t\tall\t\t\tmd5\n";
}
else
{
$new_pg_hba_conf .= $line."\n";
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_file => $update_file }});
if ($update_file)
{
$anvil->Storage->write_file({
debug => 2,
file => $anvil->data->{path}{configs}{'pg_hba.conf'},
body => $new_pg_hba_conf,
user => "postgres",
group => "postgres",
mode => "0600",
overwrite => 1,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0057", variables => { file => $anvil->data->{path}{configs}{'pg_hba.conf'} }});
}
}
# Start the daemon. '0' = started, anything else is a problem.
my $return_code = $anvil->System->start_daemon({debug => 3, daemon => $anvil->data->{sys}{daemon}{postgresql}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
if ($return_code eq "0")
{
# Started the daemon.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0059"});
# Make sure it is enabled on boot.
my $return_code = $anvil->System->enable_daemon({debug => 3, daemon => $anvil->data->{sys}{daemon}{postgresql}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
}
else
{
# Failed to start
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0094"});
$anvil->nice_exit({exit_code => 2});
}
}
# Create the .pgpass file, if needed.
my $created_pgpass = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => {
'path::secure::postgres_pgpass' => $anvil->data->{path}{secure}{postgres_pgpass},
"database::${local_uuid}::password" => $anvil->data->{database}{$local_uuid}{password},
}});
if ((not -e $anvil->data->{path}{secure}{postgres_pgpass}) && ($anvil->data->{database}{$local_uuid}{password}))
{
my $body = "*:*:*:postgres:".$anvil->data->{database}{$local_uuid}{password}."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, secure => 1, list => { body => $body }});
$anvil->Storage->write_file({
file => $anvil->data->{path}{secure}{postgres_pgpass},
body => $body,
user => "postgres",
group => "postgres",
mode => "0600",
overwrite => 1,
secure => 1,
});
if (-e $anvil->data->{path}{secure}{postgres_pgpass})
{
$created_pgpass = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { created_pgpass => $created_pgpass }});
}
}
# Does the database user exist?
my $create_user = 1;
my $database_user = $anvil->data->{database}{$local_uuid}{user} ? $anvil->data->{database}{$local_uuid}{user} : $anvil->data->{sys}{database}{user};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_user => $database_user }});
if (not $database_user)
{
# No database user defined
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0099", variables => { uuid => $local_uuid }});
$anvil->nice_exit({exit_code => 3});
}
my ($user_list, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { user_list => $user_list, return_code => $return_code }});
foreach my $line (split/\n/, $user_list)
{
if ($line =~ /^ $database_user\s+\|\s+(\d+)/)
{
# User exists already
my $id = $1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0060", variables => { user => $database_user, id => $id }});
$create_user = 0;
last;
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { create_user => $create_user }});
if ($create_user)
{
# Create the user
my ($create_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createuser}." --no-superuser --createdb --no-createrole $database_user\"", source => $THIS_FILE, line => __LINE__});
(my $user_list, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT usename, usesysid FROM pg_catalog.pg_user;'\"", source => $THIS_FILE, line => __LINE__});
my $user_exists = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_output => $create_output, user_list => $user_list }});
foreach my $line (split/\n/, $user_list)
{
if ($line =~ /^ $database_user\s+\|\s+(\d+)/)
{
# Success!
my $id = $1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0095", variables => { user => $database_user, id => $id }});
$user_exists = 1;
last;
}
}
if (not $user_exists)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0096", variables => { user => $database_user }});
$anvil->nice_exit({exit_code => 4});
}
# Update/set the passwords.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { "database::${local_uuid}::password" => $anvil->data->{database}{$local_uuid}{password} }});
if ($anvil->data->{database}{$local_uuid}{password})
{
foreach my $user ("postgres", $database_user)
{
my ($update_output, $return_code) = $anvil->System->call({secure => 1, shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c \\\"ALTER ROLE $user WITH PASSWORD '".$anvil->data->{database}{$local_uuid}{password}."';\\\"\"", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => { update_output => $update_output, return_code => $return_code }});
foreach my $line (split/\n/, $user_list)
{
if ($line =~ /ALTER ROLE/)
{
# Password set
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0100", variables => { user => $user }});
}
}
}
}
}
# Create the database, if needed.
my $create_database = 1;
my $database_name = $anvil->data->{database}{$local_uuid}{name} ? $anvil->data->{database}{$local_uuid}{name} : $anvil->data->{sys}{database}{name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_name => $database_name }});
undef $return_code;
(my $database_list, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT datname FROM pg_catalog.pg_database;'\"", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_list => $database_list, return_code => $return_code }});
foreach my $line (split/\n/, $database_list)
{
if ($line =~ /^ $database_name$/)
{
# Database already exists.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0105", variables => { database => $database_name }});
$create_database = 0;
last;
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { create_database => $create_database }});
if ($create_database)
{
my ($create_output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{createdb}." --owner ".$database_user." ".$database_name."\"", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_output => $create_output, return_code => $return_code }});
undef $return_code;
my $database_exists = 0;
(my $database_list, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{su}." - postgres -c \"".$anvil->data->{path}{exe}{psql}." template1 -c 'SELECT datname FROM pg_catalog.pg_database;'\"", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_list => $database_list, return_code => $return_code }});
foreach my $line (split/\n/, $database_list)
{
if ($line =~ /^ $database_name$/)
{
# Database created
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0110", variables => { database => $database_name }});
$database_exists = 1;
last;
}
}
if (not $database_exists)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0109", variables => { database => $database_name }});
$anvil->nice_exit({exit_code => 5});
}
}
# Remove the temporary password file.
if (($created_pgpass) && (-e $anvil->data->{path}{secure}{postgres_pgpass}))
{
unlink $anvil->data->{path}{secure}{postgres_pgpass};
if (-e $anvil->data->{path}{secure}{postgres_pgpass})
{
# Failed to unlink the file.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0107"});
}
}
#####################################################################################################
# NOTE: Below here is stuff that is for general setup. If it grows, we'll have to rename this tool. #
#####################################################################################################
### TODO: This will need to set the proper SELinux context.
# Apache run scripts can't call the system UUID, so we'll write it to a text file.
if (not -e $anvil->data->{path}{data}{host_uuid})
{
$anvil->Storage->write_file({
file => $anvil->data->{path}{data}{host_uuid},
body => $anvil->Get->host_uuid,
user => "apache",
group => "apache",
mode => "0666",
overwrite => 0,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0011", variables => { file => $anvil->data->{path}{configs}{'postgresql.conf'} }});
}
# Log level 3 creates so much logging that it hits journald's rate limiting (1000 logs per 30
# seconds). So we need to disable it.
if (not -e $anvil->data->{path}{configs}{'journald_anvil'})
{
# Write the file to disable journald rate limiting.
my $body = "# This disables the rate limiting so that when log level is set to 3, log
# entries aren't lost. If you want to override this, don't delete the file,
# just comment out the lines below.
[Journal]
RateLimitInterval=0
RateLimitBurst=0
";
$anvil->Storage->write_file({
debug => 2,
file => $anvil->data->{path}{configs}{'journald_anvil'},
body => $body,
user => "root",
group => "root",
mode => "0644",
overwrite => 0,
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0012", variables => { file => $anvil->data->{path}{configs}{'journald_anvil'} }});
my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{systemctl}." restart systemd-journald.service", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
}
}
else
{
# Didn't find an entry for this machine. This is normal on nodes.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0143"});
}
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
# This adds this machine to the local anvil.conf file.
sub add_to_local_config
{
my ($anvil) = @_;
# Write the password to a file.
my $password_file = "/tmp/striker-manage-peers.".$anvil->Get->uuid;
$anvil->Storage->write_file({
secure => 1,
file => $password_file,
body => "Initial1",
mode => "0600",
overwrite => 1,
});
# Make the shell call, and parse the output looking for our own entry
my $host_uuid = $anvil->Get->host_uuid();
my ($output, $return_code) = $anvil->System->call({
shell_call => $anvil->data->{path}{exe}{'striker-manage-peers'}." --add --host-uuid ".$anvil->Get->host_uuid." --host localhost --port 5432 --password-file ".$password_file." --ping 0",
source => $THIS_FILE,
line => __LINE__,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
# Remove the password.
unlink $password_file;
# Re-read the config and make sure we have our own entry.
sleep 1;
$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
# If we still don't have a local_uuid, something went wrong.
if (not $anvil->data->{database}{$host_uuid}{host})
{
print $anvil->Words->string({key => "error_0010"})."\n";
$anvil->nice_exit({exit_code => 1});
}
return($host_uuid);
}