* Created the new anvil-manage-host that can check/set if a host is configured. On Strikers, it can age out data, resync data, and check/set if the local database is active.

* Updated striker-prep-database to again enable the postgresql service.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 3 years ago
parent b575b1ebc1
commit 74b7719cf5
  1. 48
      Anvil/Tools/Database.pm
  2. 1
      Anvil/Tools/System.pm
  3. 52
      share/words.xml
  4. 7
      tools/anvil-daemon
  5. 310
      tools/anvil-manage-host
  6. 5
      tools/striker-prep-database

@ -1197,6 +1197,10 @@ This module will return the number of databases that were successfully connected
Parameters;
=head3 all (optional, default 0)
If set, then the checks to see if a database is active or not is ignored. If the postgresql server is running, we will connect to it.
=head3 check_for_resync (optional, default 0)
If set to C<< 1 >>, and there are 2 or more databases available, a check will be make to see if the databases need to be resync'ed or not. This is also set if the command line switch C<< --resync-db >> is used.
@ -1281,6 +1285,7 @@ sub connect
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->connect()" }});
my $all = defined $parameter->{all} ? $parameter->{all} : 0;
my $check_if_configured = defined $parameter->{check_if_configured} ? $parameter->{check_if_configured} : 0;
my $db_uuid = defined $parameter->{db_uuid} ? $parameter->{db_uuid} : "";
my $check_for_resync = defined $parameter->{check_for_resync} ? $parameter->{check_for_resync} : 0;
@ -1292,6 +1297,7 @@ sub connect
my $tables = defined $parameter->{tables} ? $parameter->{tables} : "";
my $test_table = defined $parameter->{test_table} ? $parameter->{test_table} : $anvil->data->{sys}{database}{test_table};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
all => $all,
check_if_configured => $check_if_configured,
db_uuid => $db_uuid,
check_for_resync => $check_for_resync,
@ -1682,7 +1688,9 @@ sub connect
"cache::database_handle::${uuid}" => $anvil->data->{cache}{database_handle}{$uuid},
}});
# Before I continue, see if this database is going offline.
# Before I continue, see if this database is inactive (unless 'all' is set).
if (not $all)
{
my ($active_value, undef, undef) = $anvil->Database->read_variable({
debug => $debug,
uuid => $uuid,
@ -1727,6 +1735,7 @@ sub connect
next;
}
}
}
# Still here? We're active
$anvil->data->{db_status}{$uuid}{active} = 1;
@ -1776,13 +1785,17 @@ sub connect
$anvil->data->{sys}{database}{read_uuid} = $uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::read_uuid" => $anvil->data->{sys}{database}{read_uuid} }});
# If we're being called with 'all', don't set active as we could be just checking if
# we're active or not.
if (not $all)
{
# If we're a striker, set the variable saying we're active if we need to.
my ($active_value, undef, undef) = $anvil->Database->read_variable({
debug => $debug,
uuid => $uuid,
variable_name => "database::".$uuid."::active",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active_value => $active_value }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { active_value => $active_value }});
if (not $active_value)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0698"});
@ -1797,6 +1810,7 @@ sub connect
});
}
}
}
elsif (not $anvil->data->{sys}{database}{read_uuid})
{
$anvil->data->{sys}{database}{read_uuid} = $uuid;
@ -1841,6 +1855,10 @@ sub connect
}
# If we're a striker and no connections were found, start our database.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
local_host_type => $local_host_type,
"sys::database::connections" => $anvil->data->{sys}{database}{connections},
}});
if (($local_host_type eq "striker") && (not $anvil->data->{sys}{database}{connections}))
{
# Tell the user we're going to try to load and start.
@ -1854,7 +1872,7 @@ sub connect
my $db_name = "anvil";
my $dump_files = [];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { directory => $directory }});
local(*DIRECTORY);
opendir(DIRECTORY, $directory);
while(my $file = readdir(DIRECTORY))
@ -1982,7 +2000,7 @@ sub connect
# Check if the dameon is running
my $running = $anvil->System->check_daemon({daemon => $anvil->data->{sys}{daemon}{postgresql}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { running => $running }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { running => $running }});
if (not $running)
{
my $return_code = $anvil->System->start_daemon({daemon => $anvil->data->{sys}{daemon}{postgresql}});
@ -12245,15 +12263,15 @@ sub insert_or_update_states
}
foreach my $db_uuid (@{$db_uuids})
{
my $count = $anvil->Database->query({debug => 2, uuid => $db_uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
my $count = $anvil->Database->query({uuid => $db_uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's2:db_uuid' => $db_uuid,
's2:count' => $count,
}});
if (not $count)
{
$hosts_ok = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts_ok => $hosts_ok }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { hosts_ok => $hosts_ok }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0144", variables => {
state_info => $state_name." -> ".$state_note,
@ -17143,13 +17161,25 @@ sub _age_out_data
uuid_column => $uuid_column,
}});
# Make sure the table exists, skip it if not.
my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->Database->quote($child_table)." AND schemaname='public';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if (not $count)
{
# Table doesn't exist yet, skip it.
next;
}
# Get a list of all records.
my $queries = [];
my $query = "SELECT ".$uuid_column." FROM ".$child_table.";";
$query = "SELECT ".$uuid_column." FROM ".$child_table.";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,

@ -592,6 +592,7 @@ sub check_if_configured
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->check_if_configured()" }});
my ($configured, $variable_uuid, $modified_date) = $anvil->Database->read_variable({
debug => $debug,
variable_name => "system::configured",
variable_source_uuid => $anvil->Get->host_uuid,
variable_source_table => "hosts",

@ -503,6 +503,7 @@ The output, if any, was;
<key name="error_0355">Failed to load the database file: [#!variable!file!#]. Deleting it so it's not considered in the next load attempt.</key>
<key name="error_0356">Failed to read the kernel release on the host: [#!variable!target!#]. The return code was: [#!variable!return_code!#] (expected '0') and the release output, if any, was: [#!variable!output!#].</key>
<key name="error_0357">The program: [#!variable!program!#] is using: [#!variable!ram_used!#] (#!variable!ram_used_bytes!# Bytes). This is probably caused by a memory leak, so we will now exit so that systemctl can restart us. If this is happening repeatedly, please contact support.</key>
<key name="error_0358">This is not a Striker host.</key>
<!-- Files templates -->
<!-- NOTE: Translating these files requires an understanding of which lines are translatable -->
@ -2457,6 +2458,57 @@ Are you sure that you want to delete the server: [#!variable!server_name!#]? [Ty
<key name="message_0267">Preparing to manage DR for a server.</key>
<key name="message_0268">UUID Column counts for: [history.#!variable!table!#]:</key>
<key name="message_0269">Counting entries for each unique: [#!variable!column!#] in the table [#!variable!table!#]. Please be patient.</key>
<key name="message_0270">Marking this host as configured.</key>
<key name="message_0271">This host is already marked as configured.</key>
<key name="message_0272">Marking this host as configured.</key>
<key name="message_0273">This host is already marked as configured.</key>
<key name="message_0274">This host is marked as unconfigured.</key>
<key name="message_0275">This host is marked as configured.</key>
<key name="message_0276">This database is marked as inactive.</key>
<key name="message_0277">This database is marked as active.</key>
<key name="message_0278">Marking this database as active.</key>
<key name="message_0279">This database is already marked as active.</key>
<key name="message_0280">Marking this database as inactive.</key>
<key name="message_0281">This database is already marked as inactive.</key>
<key name="message_0282">
Available options;
--age-out-database
This purches older records to reduce the size of the database.
--check-configured
This checks to see if the host is marked as configured or not.
--check-database
This checks to see if the host's database is marked as active or not.
--database-active
This marks the host's database as active.
--database-inactive
This marks the host's database as inactive.
--mark-configured
This marks the host as being configured.
--mark-unconfigured
This marks the host as being unconfigured.
--resync-database
Force a resync of the databases.
</key>
<key name="message_0283">I was asked to resync, but there is only one database available so no sense proceeding.</key>
<key name="message_0284">I was asked to resync. Disconnecting from databases and reconnecting with resync.</key>
<key name="message_0285">Aging out data to thin down the database(s).</key>
<key name="message_0286">Prior to resync, we will check to see if any scan agent schemas need to be loaded.</key>
<!-- Success messages shown to the user -->
<key name="ok_0001">Saved the mail server information successfully!</key>

@ -625,6 +625,8 @@ sub handle_periodic_tasks
# down our database.
if ($host_type eq "striker")
{
### NOTE: Database shutdown logic is disabled. Too flaky.
=cut
if ($anvil->data->{sys}{database}{connections} > 1)
{
# Make sure that all active databases are in the host's table. If they're
@ -689,6 +691,7 @@ sub handle_periodic_tasks
}
}
}
=cut
# If we're the active database, dump our database out and rsync it to our peers.
my $peers = keys %{$anvil->data->{database}};
@ -768,6 +771,10 @@ sub handle_periodic_tasks
# Archive old data
$anvil->Database->archive_database();
### TODO: This is here only to handle the period of time where we disabled postgres
### on boot. This should be removed sometime after 2022-08-01
$anvil->System->enable_daemon({daemon => $anvil->data->{sys}{daemon}{postgresql}});
# Record a job, don't call it directly. It takes too long to run.
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE,

@ -0,0 +1,310 @@
#!/usr/bin/perl
#
# This tool provides a way to perform actions on a host that normally are handled by other processes, like
# marking a system as configure, etc. This will likely grow constantly over time.
#
# Supported switches;
# --mark-configured, --mark-unconfigured
# - Set / unset the 'system::configured' flag for the host.
# --database-active, --database-inactive
# - On Striker, mark the local database as active / inactive
#
use strict;
use warnings;
use Anvil::Tools;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
# Read switches
$anvil->data->{switches}{'age-out-database'} = "";
$anvil->data->{switches}{'check-configured'} = "";
$anvil->data->{switches}{'check-database'} = "";
$anvil->data->{switches}{'database-active'} = "";
$anvil->data->{switches}{'database-inactive'} = "";
$anvil->data->{switches}{'mark-configured'} = "";
$anvil->data->{switches}{'mark-unconfigured'} = "";
$anvil->data->{switches}{'resync-database'} = "";
$anvil->Get->switches;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'switches::age-out-database' => $anvil->data->{switches}{'age-out-database'},
'switches::check-configured' => $anvil->data->{switches}{'check-configured'},
'switches::check-database' => $anvil->data->{switches}{'check-database'},
'switches::database-active' => $anvil->data->{switches}{'database-active'},
'switches::database-inactive' => $anvil->data->{switches}{'database-inactive'},
'switches::mark-configured' => $anvil->data->{switches}{'mark-configured'},
'switches::mark-unconfigured' => $anvil->data->{switches}{'mark-unconfigured'},
'switches::resync-database' => $anvil->data->{switches}{'resync-database'},
}});
$anvil->Database->connect({all => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, we can't do anything.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0077"});
$anvil->nice_exit({exit_code => 1});
}
if (($anvil->data->{switches}{'mark-configured'}) or ($anvil->data->{switches}{'mark-unconfigured'}) or ($anvil->data->{switches}{'check-configured'}))
{
update_config($anvil);
}
elsif (($anvil->data->{switches}{'database-active'}) or ($anvil->data->{switches}{'database-inactive'}) or ($anvil->data->{switches}{'check-database'}))
{
update_database($anvil);
}
elsif ($anvil->data->{switches}{'age-out-database'})
{
age_out_data($anvil);
}
elsif ($anvil->data->{switches}{'resync-database'})
{
resync_database($anvil);
}
else
{
# Show the options.
print $anvil->Words->string({key => "message_0282"})."\n";
}
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
sub age_out_data
{
my ($anvil) = @_;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0285"});
$anvil->Database->_age_out_data({debug => 2});
return(0);
}
sub resync_database
{
my ($anvil) = @_;
if ($anvil->data->{sys}{database}{connections} == 1)
{
# No reason to proceed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0283"});
return(0);
}
# Before we reconnect, make sure all scan agent tables are loaded.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0286"});
$anvil->ScanCore->_scan_directory({directory => $anvil->data->{path}{directories}{scan_agents}});
my $tables = "";
foreach my $agent_name (sort {$a cmp $b} keys %{$anvil->data->{scancore}{agent}})
{
my $agent_path = $anvil->data->{scancore}{agent}{$agent_name};
my $agent_words = $agent_path.".xml";
my $schema_file = $agent_path.".sql";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
agent_name => $agent_name,
agent_path => $agent_path,
agent_words => $agent_words,
schema_file => $schema_file,
}});
if (-e $schema_file)
{
# See that it's loaded.
$tables = $anvil->Database->get_tables_from_schema({schema_file => $schema_file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { tables => $tables }});
foreach my $table (@{$tables})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { table => $table }});
}
$anvil->Database->check_agent_data({
agent => $agent_name,
tables => $tables,
});
}
}
# Disconnect and reconnect.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0284"});
$anvil->Database->disconnect();
$anvil->refresh();
$anvil->Database->connect({
all => 1,
check_for_resync => 1,
});
return(0);
}
sub update_database
{
my ($anvil) = @_;
# Are we a striker?
my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type ne "striker")
{
# Nope.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "error_0358"});
$anvil->nice_exit({exit_code => 1});
}
my $host_uuid = $anvil->Get->host_uuid();
my $variable_name = "database::".$host_uuid."::active";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
host_uuid => $host_uuid,
variable_name => $variable_name,
}});
# Read if it's active or inactive yet.
my ($active_value, undef, undef) = $anvil->Database->read_variable({variable_name => "database::".$host_uuid."::active"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active_value => $active_value }});
if ($anvil->data->{switches}{'check-database'})
{
if ($active_value)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0277"});
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0276"});
}
return(0);
}
# Are we enabling or disabling the database?
if ($active_value)
{
if ($anvil->data->{switches}{'database-active'})
{
# Already active.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0279"});
}
else
{
# Mark inactive
my $variable_uuid = $anvil->Database->insert_or_update_variables({
variable_name => $variable_name,
variable_value => "0",
variable_default => "0",
variable_description => "striker_0294",
variable_section => "database",
variable_source_uuid => "NULL",
variable_source_table => "",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0280"});
}
}
else
{
if ($anvil->data->{switches}{'database-inactive'})
{
# Already inactive
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0281"});
}
else
{
# Mark active
my $variable_uuid = $anvil->Database->insert_or_update_variables({
variable_name => $variable_name,
variable_value => "1",
variable_default => "0",
variable_description => "striker_0294",
variable_section => "database",
variable_source_uuid => "NULL",
variable_source_table => "",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0278"});
}
}
return(0);
}
sub update_config
{
my ($anvil) = @_;
# Check if it's already configured.
my $configured = $anvil->System->check_if_configured({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configured => $configured }});
if ($anvil->data->{switches}{'check-configured'})
{
if ($configured)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0275"});
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0274"});
}
return(0);
}
if ($configured)
{
if ($anvil->data->{switches}{'mark-configured'})
{
# Nothing to do.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0271"});
}
else
{
# Mark UN configures.
my $variable_uuid = $anvil->Database->insert_or_update_variables({
variable_name => "system::configured",
variable_value => 0,
variable_default => "",
variable_description => "striker_0048",
variable_section => "system",
variable_source_uuid => $anvil->Get->host_uuid,
variable_source_table => "hosts",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0272"});
}
}
else
{
if ($anvil->data->{switches}{'mark-unconfigured'})
{
# Nothing to do.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0273"});
}
else
{
# Mark configures.
my $variable_uuid = $anvil->Database->insert_or_update_variables({
variable_name => "system::configured",
variable_value => 1,
variable_default => "",
variable_description => "striker_0048",
variable_section => "system",
variable_source_uuid => $anvil->Get->host_uuid,
variable_source_table => "hosts",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0270"});
}
}
return(0);
}

@ -237,9 +237,8 @@ if ($local_uuid)
# Started the daemon.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0059"});
### NOTE: We no longer enable postgres on boot. When the first call is made to
### Database->connect on a striker, and no databases are available, it will
### start up the local daemon then.
# Enable the daemon.
$anvil->System->enable_daemon({debug => 2, daemon => $anvil->data->{sys}{daemon}{postgresql}});
}
else
{

Loading…
Cancel
Save