* Fixed a bug where setting the debug level to 3 caused a deep recursion and a system hang.

* Update Anvil::Tools->new() to access the parameters 'log_level', 'log_secure' and 'debug', streamlining the frequent calls to $anvil->Log->level and ->secure in program startup, and allowing the values to take effect during the ->new constructor.
* Passed 'debug' to child method calls in more places (still more to do though).
* Fixed a bug where 'test_table' wasn't set in the right place, causing the database to try to initialize repeatedly.
* Made Database->archive_database only run if called with root access.
* Now the number of database connections are stored in 'sys::db_connections' instead of checking the returned number, and that is cleared on disconnect.
* Started working more on 'anvil-daemon', including adding support for System->call being taking 'background', 'stderr_file' and 'stdout_file' paramters which, when set, used Proc::Simple to background the process.
* Did some more work on database archiving, though still far from done.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent a364141d81
commit 0fa3c42f2f
  1. 77
      Anvil/Tools.pm
  2. 105
      Anvil/Tools/Database.pm
  3. 21
      Anvil/Tools/Get.pm
  4. 5
      Anvil/Tools/Log.pm
  5. 78
      Anvil/Tools/System.pm
  6. 14
      anvil.conf
  7. 11
      cgi-bin/striker
  8. 2
      html/index.html
  9. 1
      rpm/SPECS/anvil.spec
  10. 3
      share/words.xml
  11. 10
      tools/anvil-change-password
  12. 14
      tools/anvil-configure-striker
  13. 163
      tools/anvil-daemon
  14. 3
      tools/anvil-jobs
  15. 4
      tools/anvil-prep-database
  16. 3
      tools/anvil-scan-network
  17. 13
      tools/anvil-update-states

@ -147,22 +147,6 @@ sub new
my $anvil = $self;
weaken($anvil); # Helps avoid memory leaks. See Scalar::Utils
# Record the start time.
$anvil->data->{ENV_VALUES}{START_TIME} = Time::HiRes::time;
# Set passed parameters if needed.
if (ref($parameter) eq "HASH")
{
### TODO: Calls to allow the user to override defaults...
# Local parameters...
}
elsif ($parameter)
{
# Um...
print $THIS_FILE." ".__LINE__."; Anvil::Tools->new() invoked with an invalid parameter. Expected a hash reference, but got: [$parameter]\n";
exit(1);
}
# Get a handle on the various submodules
$anvil->Account->parent($anvil);
$anvil->Alert->parent($anvil);
@ -178,8 +162,38 @@ sub new
$anvil->Validate->parent($anvil);
# Set some system paths and system default variables
$anvil->_set_paths;
$anvil->_set_defaults;
$anvil->_set_paths();
$anvil->_set_defaults();
# Record the start time.
$anvil->data->{ENV_VALUES}{START_TIME} = Time::HiRes::time;
# Set passed parameters if needed.
my $debug = 3;
if (ref($parameter) eq "HASH")
{
# Local parameters...
if ($parameter->{debug})
{
$debug = $parameter->{debug};
}
if ($parameter->{log_secure})
{
$anvil->Log->secure({set => $parameter->{log_secure}});
}
}
elsif ($parameter)
{
# Um...
print $THIS_FILE." ".__LINE__."; Anvil::Tools->new() invoked with an invalid parameter. Expected a hash reference, but got: [$parameter]\n";
exit(1);
}
# If the user passed a custom log level, sit it now.
if ($parameter->{log_level})
{
$anvil->Log->level({set => $parameter->{log_level}});
}
# This will help clean up if we catch a signal.
$SIG{INT} = sub { $anvil->catch_sig({signal => "INT"}); };
@ -202,22 +216,33 @@ sub new
$anvil->data($parameter->{data}) if $parameter->{data};
# Initialize the list of directories to seach.
$anvil->Storage->search_directories({initialize => 1});
$anvil->Storage->search_directories({debug => $debug, initialize => 1});
# I need to read the initial words early.
$anvil->Words->read({file => $anvil->data->{path}{words}{'words.xml'}});
$anvil->Words->read({debug => $debug, file => $anvil->data->{path}{words}{'words.xml'}});
# If the local './tools.conf' file exists, read it in.
if (-r $anvil->data->{path}{configs}{'anvil.conf'})
{
$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
$anvil->Storage->read_config({debug => $debug, file => $anvil->data->{path}{configs}{'anvil.conf'}});
### TODO: Should anvil.conf override parameters?
# Let parameters override config file values.
if ($parameter->{log_level})
{
$anvil->Log->level({set => $parameter->{log_level}});
}
if ($parameter->{log_secure})
{
$anvil->Log->secure({set => $parameter->{log_secure}});
}
}
# Get the local host UUID.
$anvil->Get->host_uuid;
$anvil->Get->host_uuid({debug => $debug});
# Read in any command line switches.
$anvil->Get->switches;
$anvil->Get->switches({debug => $debug});
# Read in the local Anvil! version.
#...
@ -263,7 +288,7 @@ Data can be entered into or access by treating '$anvil->data' as a normal hash r
# Set 'A thing' in 'foo'.
$anvil->data->{foo} = "A thing";
The C<$an> variable is set inside all modules and acts as shared storage for variables, values and references in all modules. It acts as the core storage for most applications using Anvil::Tools.
The C<< $anvil >> variable is set inside all modules and acts as shared storage for variables, values and references in all modules. It acts as the core storage for most applications using Anvil::Tools.
=cut
sub data
@ -717,6 +742,7 @@ sub _set_defaults
log_transactions => 0,
maximum_batch_size => 25000,
name => "anvil",
test_table => "hosts",
user => "admin",
},
host_type => "",
@ -806,9 +832,6 @@ sub _set_defaults
server => "",
tag => "anvil",
},
sql => {
test_table => "hosts",
},
template => {
html => "alteeve",
},

@ -116,10 +116,25 @@ sub archive_database
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->archive_database()" }});
# Is archiving disabled?
if (not $anvil->data->{sys}{database}{archive}{trigger})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0189"});
return(1);
}
# Only the root user can archive the database so that the archived files can be properly secured.
if (($< != 0) && ($> != 0))
{
# Not root
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0188"});
return(1);
}
# If we don't have an array of tables, we have nothing to do.
if ((not exists $anvil->data->{sys}{database}{check_tables}) or (ref(@{$anvil->data->{sys}{database}{check_tables}} ne "ARRAY")))
{
return(0);
return(1);
}
# We'll use the list of tables created for _find_behind_databases()'s 'sys::database::check_tables'
@ -196,11 +211,11 @@ sub check_lock_age
=head2 configure_pgsql
This configures the local database server. Specifically, it checks to make sure the daemon is running and starts it if not. It also checks the 'pg_hba.conf' configuration to make sure it is set properly to listen on this machine's IP addresses and interfaces.
This configures the local database server. Specifically, it checks to make sure the daemon is running and starts it if not. It also checks the C<< pg_hba.conf >> configuration to make sure it is set properly to listen on this machine's IP addresses and interfaces.
If the system is already configured, this method will do nothing, so it is safe to call it at any time.
If there is a problem, C<< !!error!! >> is returned.
If the method completes, C<< 0 >> is returned. If this method is called without C<< root >> access, it returns C<< 1 >> without doing anything. If there is a problem, C<< !!error!! >> is returned.
=cut
sub configure_pgsql
@ -221,11 +236,11 @@ sub configure_pgsql
# This is a minor error as it will be hit by every unpriviledged program that connects to the
# database(s).
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, priority => "alert", key => "log_0113"});
return("!!error!!");
return(1);
}
# First, is it running?
my $running = $anvil->System->check_daemon({daemon => "postgresql"});
my $running = $anvil->System->check_daemon({debug => $debug, daemon => "postgresql"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { running => $running }});
if (not $running)
@ -234,7 +249,7 @@ sub configure_pgsql
if (not -e $anvil->data->{path}{configs}{'pg_hba.conf'})
{
# Initialize.
my $output = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
my $output = $anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{'postgresql-setup'}." initdb", source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { output => $output }});
# Did it succeed?
@ -250,14 +265,14 @@ sub configure_pgsql
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0055"});
# Enable it on boot.
my $return_code = $anvil->System->enable_daemon({daemon => "postgresql"});
my $return_code = $anvil->System->enable_daemon({debug => $debug, daemon => "postgresql"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { return_code => $return_code }});
}
}
}
# Setup postgresql.conf, if needed
my $postgresql_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'postgresql.conf'}});
my $postgresql_conf = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{configs}{'postgresql.conf'}});
my $update_postgresql_file = 1;
my $new_postgresql_conf = "";
foreach my $line (split/\n/, $postgresql_conf)
@ -288,6 +303,7 @@ sub configure_pgsql
if (not -e $postgresql_backup)
{
$anvil->Storage->copy_file({
debug => $debug,
source_file => $anvil->data->{path}{configs}{'postgresql.conf'},
target_file => $postgresql_backup,
});
@ -295,6 +311,7 @@ sub configure_pgsql
# Write the updated one.
$anvil->Storage->write_file({
debug => $debug,
file => $anvil->data->{path}{configs}{'postgresql.conf'},
body => $new_postgresql_conf,
user => "postgres",
@ -306,7 +323,7 @@ sub configure_pgsql
}
# Setup pg_hba.conf now, if needed.
my $pg_hba_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'pg_hba.conf'}});
my $pg_hba_conf = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{configs}{'pg_hba.conf'}});
my $update_pg_hba_file = 1;
my $new_pg_hba_conf = "";
foreach my $line (split/\n/, $pg_hba_conf)
@ -631,7 +648,7 @@ sub connect
my $source = defined $parameter->{source} ? $parameter->{source} : "core";
my $sql_file = defined $parameter->{sql_file} ? $parameter->{sql_file} : $anvil->data->{path}{sql}{'anvil.sql'};
my $tables = defined $parameter->{tables} ? $parameter->{tables} : "";
my $test_table = defined $parameter->{test_table} ? $parameter->{test_table} : $anvil->data->{defaults}{sql}{test_table};
my $test_table = defined $parameter->{test_table} ? $parameter->{test_table} : $anvil->data->{sys}{database}{test_table};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
source => $source,
sql_file => $sql_file,
@ -730,6 +747,7 @@ sub connect
{
# Can I ping?
my ($pinged) = $anvil->System->ping({
debug => $debug,
ping => $host,
count => 1,
timeout => $anvil->data->{database}{$uuid}{ping},
@ -754,7 +772,7 @@ sub connect
}
# Before we try to connect, see if this is a local database and, if so, make sure it's setup.
my $is_local = $anvil->System->is_local({host => $host});
my $is_local = $anvil->System->is_local({debug => $debug, host => $host});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { is_local => $is_local }});
if ($is_local)
{
@ -762,7 +780,7 @@ sub connect
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::read_db_uuid" => $anvil->data->{sys}{read_db_uuid} }});
# Set it up (or update it) if needed. This method just returns if nothing is needed.
$anvil->Database->configure_pgsql({uuid => $uuid});
$anvil->Database->configure_pgsql({debug => $debug, uuid => $uuid});
}
elsif (not $anvil->data->{sys}{read_db_uuid})
{
@ -775,6 +793,7 @@ sub connect
if (not $is_local)
{
my $remote_version = $anvil->Get->anvil_version({
debug => $debug,
target => $host,
password => $password,
});
@ -783,7 +802,7 @@ sub connect
"anvil->_anvil_version" => $anvil->_anvil_version,
}});
if ($remote_version ne $anvil->_anvil_version)
if ($remote_version ne $anvil->_anvil_version({debug => $debug}))
{
# Version doesn't match,
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0145", variables => {
@ -886,13 +905,13 @@ sub connect
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'sys::use_db_fh' => $anvil->data->{sys}{use_db_fh} }});
}
# If the '$test_table' isn't the same as 'defaults::sql::test_table', see if the core schema needs loading first.
if ($test_table ne $anvil->data->{defaults}{sql}{test_table})
# If the '$test_table' isn't the same as 'sys::database::test_table', see if the core schema needs loading first.
if ($test_table ne $anvil->data->{sys}{database}{test_table})
{
my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{defaults}{sql}{test_table})." AND schemaname='public';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if ($count < 1)
@ -900,7 +919,7 @@ sub connect
### TODO: Create a version file/flag and don't sync with peers unless
### they are the same version. Back-port this to v2.
# Need to load the database.
$anvil->Database->initialize({uuid => $uuid, sql_file => $anvil->data->{path}{sql}{'anvil.sql'}});
$anvil->Database->initialize({debug => $debug, uuid => $uuid, sql_file => $anvil->data->{path}{sql}{'anvil.sql'}});
}
}
@ -908,13 +927,13 @@ sub connect
my $query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename=".$anvil->data->{sys}{use_db_fh}->quote($test_table)." AND schemaname='public';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if ($count < 1)
{
# Need to load the database.
$anvil->Database->initialize({uuid => $uuid, sql_file => $sql_file});
$anvil->Database->initialize({debug => $debug, uuid => $uuid, sql_file => $sql_file});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
@ -952,7 +971,7 @@ sub connect
my $query = "SELECT cast(now() AS timestamp with time zone)::timestamptz(0);";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->data->{sys}{db_timestamp} = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->data->{sys}{db_timestamp} = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::db_timestamp" => $anvil->data->{sys}{db_timestamp} }});
}
@ -1002,6 +1021,7 @@ sub connect
# If I've not sent an alert about this DB loss before, send one now.
my $set = $anvil->Alert->check_alert_sent({
debug => $debug,
type => "set",
set_by => $THIS_FILE,
record_locator => $uuid,
@ -1025,6 +1045,7 @@ sub connect
# These are warning level alerts.
$anvil->Alert->register_alert({
debug => $debug,
alert_level => "warning",
alert_set_by => $THIS_FILE,
alert_title_key => "alert_title_0003",
@ -1055,12 +1076,13 @@ sub connect
# my $query = "SELECT COUNT(*) FROM hosts WHERE host_name = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{database}{$uuid}{host}).";";
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
#
# my $count = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
# my $count = $anvil->Database->query({uuid => $uuid, debug => $debug, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
#
# if ($count > 0)
# {
my $cleared = $anvil->Alert->check_alert_sent({
debug => $debug,
type => "clear",
set_by => $THIS_FILE,
record_locator => $uuid,
@ -1070,6 +1092,7 @@ sub connect
if ($cleared)
{
$anvil->Alert->register_alert({
debug => $debug,
level => "warning",
agent_name => "Anvil!",
title_key => "an_title_0006",
@ -1091,7 +1114,7 @@ sub connect
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0103"});
# Disconnect and set the connection count to '0'.
$anvil->Database->disconnect();
$anvil->Database->disconnect({debug => $debug});
$connections = 0;
}
@ -1100,25 +1123,29 @@ sub connect
if ($connections > 1)
{
$anvil->Database->_find_behind_databases({
debug => $debug,
source => $source,
tables => $tables,
});
}
# Hold if a lock has been requested.
$anvil->Database->locking();
$anvil->Database->locking({debug => $debug});
# Mark that we're not active.
$anvil->Database->mark_active({set => 1});
$anvil->Database->mark_active({debug => $debug, set => 1});
# Archive old data.
$anvil->Database->archive_database({});
$anvil->Database->archive_database({debug => $debug});
# Sync the database, if needed.
$anvil->Database->resync_databases;
$anvil->Database->resync_databases({debug => $debug});
# Add ourselves to the database, if needed.
$anvil->Database->insert_or_update_hosts;
$anvil->Database->insert_or_update_hosts({debug => $debug});
# We'll store the number of connections in the hash.
$anvil->data->{sys}{db_connections} = $connections;
return($connections);
}
@ -1159,6 +1186,9 @@ sub disconnect
delete $anvil->data->{sys}{use_db_fh};
delete $anvil->data->{sys}{read_db_uuid};
# Set the connection count to 0.
$anvil->data->{sys}{db_connections} = 0;
return(0);
}
@ -5160,20 +5190,6 @@ sub _archive_table
return("!!error!!");
}
# Has the user disabled archiving?
my $trigger = $anvil->data->{sys}{database}{archive}{trigger};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { trigger => $trigger }});
if ((exists $anvil->data->{sys}{database}{archive}{tables}{$table}{division}) && ($anvil->data->{sys}{database}{archive}{tables}{$table}{division} =~ /^\d+$/))
{
$trigger = $anvil->data->{sys}{database}{archive}{tables}{$table}{division};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { trigger => $trigger }});
}
if ($trigger)
{
# Archiving is disabled.
return(0);
}
# First, if this table doesn't have a history schema, exit.
my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->data->{sys}{use_db_fh}->quote($table).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
@ -5191,8 +5207,11 @@ sub _archive_table
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if ($count <= $trigger)
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:count" => $count,
"s2:sys::database::archive::trigger" => $anvil->data->{sys}{database}{archive}{trigger},
}});
if ($count <= $anvil->data->{sys}{database}{archive}{trigger})
{
# History table doesn't exist, we're done.
return(0);

@ -462,25 +462,17 @@ sub host_uuid
if ($set)
{
$anvil->data->{HOST}{UUID} = $set;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "HOST::UUID" => $anvil->data->{HOST}{UUID} }});
}
elsif (not $anvil->data->{HOST}{UUID})
{
# Read dmidecode if I am root, and the cache if not.
# Read dmidecode if I am root, otherwise, read the cache.
my $uuid = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { '$<' => $<, '$>' => $> }});
if (($< == 0) or ($> == 0))
{
my $shell_call = $anvil->data->{path}{exe}{dmidecode}." --string system-uuid";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
open(my $file_handle, $shell_call." 2>&1 |") or warn $THIS_FILE." ".__LINE__."; [ Warning ] - Failed to call: [".$shell_call."], the error was: $!\n";
while(<$file_handle>)
{
# This should never be hit...
chomp;
$uuid = lc($_);
#print $THIS_FILE." ".__LINE__."; [ Debug ] - UUID: [$uuid]\n";
}
close $file_handle;
$uuid = lc($anvil->System->call({debug => $debug, shell_call => $anvil->data->{path}{exe}{dmidecode}." --string system-uuid"}));
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
}
else
{
@ -495,7 +487,7 @@ sub host_uuid
}
else
{
$uuid = $anvil->Storage->read_file({ file => $anvil->data->{path}{data}{host_uuid} });
$uuid = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{data}{host_uuid}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
}
}
@ -509,6 +501,7 @@ sub host_uuid
# Apache run scripts can't call the system UUID, so we'll write it to a text
# file.
$anvil->Storage->write_file({
debug => $debug,
file => $anvil->data->{path}{data}{host_uuid},
body => $uuid,
user => "apache",
@ -524,6 +517,7 @@ sub host_uuid
# Bad UUID.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0134", variables => { uuid => $uuid }});
$anvil->data->{HOST}{UUID} = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "HOST::UUID" => $anvil->data->{HOST}{UUID} }});
}
}
@ -534,6 +528,7 @@ sub host_uuid
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::host_uuid" => $anvil->data->{sys}{host_uuid} }});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "HOST::UUID" => $anvil->data->{HOST}{UUID} }});
return($anvil->data->{HOST}{UUID});
}

@ -173,7 +173,6 @@ This is the 'verbose' log level. It will generally generate a significant amount
This is the highest log level, and it will generate a tremendous amount of log entries. This is generally used is loops or recursive functions where the output is significant, but the usefulness of the output is not.
=head3 line (optional)
When set, the string is prepended to the log entry, after 'C<< file >> if set, and should be set to C<< __LINE__ >>. It is used to show where in 'C<< file >>' the log entry was made and can assist with debugging.
@ -317,8 +316,10 @@ sub entry
my $log_file = $anvil->data->{sys}{log_file} =~ /^\// ? $anvil->data->{sys}{log_file} : "/var/log/".$anvil->data->{sys}{log_file};
my ($directory, $file) = ($log_file =~ /^(\/.*)\/(.*)$/);
### WARNING: We MUST set the debug level really high, or else we'll go into a deep
### recursion!
# Make sure the log directory exists.
$anvil->Storage->make_directory({directory => $directory, group => 755});
$anvil->Storage->make_directory({debug => 99, directory => $directory, mode => 755});
# Now open the log
my $shell_call = $log_file;

@ -9,6 +9,7 @@ use Data::Dumper;
use Net::SSH2;
use Scalar::Util qw(weaken isweak);
use Time::HiRes qw(gettimeofday tv_interval);
use Proc::Simple;
our $VERSION = "3.0.0";
my $THIS_FILE = "System.pm";
@ -100,6 +101,10 @@ This method makes a system call and returns the output (with the last new-line r
Parameters;
=head3 background (optional, default '0')
If set to C<< 1 >>, the program will be started in the background and the C<< Proc::Simple >> handle will be returned instead of the command's output.
=head3 line (optional)
This is the line number of the source file that called this method. Useful for logging and debugging.
@ -120,6 +125,18 @@ This is the shell command to call.
This is the name of the source file calling this method. Useful for logging and debugging.
=head3 stderr_file (optional)
B<NOTE>: This is only used when C<< background >> is set to C<< 1 >>.
If set, the C<< STDERR >> output will be sent to the corresponding file. If this isn't a full path, the file will be placed under C<< /tmp/ >>.
=head3 stdout_file (optional)
B<NOTE>: This is only used when C<< background >> is set to C<< 1 >>.
If set, the C<< STDOUT >> output will be sent to the corresponding file. If this isn't a full path, the file will be placed under C<< /tmp/ >>.
=cut
sub call
{
@ -128,16 +145,22 @@ sub call
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
my $background = defined $parameter->{background} ? $parameter->{background} : 0;
my $line = defined $parameter->{line} ? $parameter->{line} : __LINE__;
my $redirect_stderr = defined $parameter->{redirect_stderr} ? $parameter->{redirect_stderr} : 1;
my $shell_call = defined $parameter->{shell_call} ? $parameter->{shell_call} : "";
my $secure = defined $parameter->{secure} ? $parameter->{secure} : 0;
my $source = defined $parameter->{source} ? $parameter->{source} : $THIS_FILE;
my $stderr_file = defined $parameter->{stderr_file} ? $parameter->{stderr_file} : "";
my $stdout_file = defined $parameter->{stdout_file} ? $parameter->{stdout_file} : "";
my $redirect = $redirect_stderr ? " 2>&1" : "";
$anvil->Log->variables({source => $source, line => $line, level => $debug, secure => $secure, list => {
background => $background,
shell_call => $shell_call,
redirect => $redirect,
redirect_stderr => $redirect_stderr,
stderr_file => $stderr_file,
stdout_file => $stdout_file,
}});
my $output = "#!error!#";
@ -175,8 +198,60 @@ sub call
if ($found)
{
# Make the system call
$output = "";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, key => "log_0011", variables => { shell_call => $shell_call }});
### TODO: We should split the arguments off, which the below does, to pass arguments
### to the shell without having the shell expand the arguments
my $program = $shell_call;
my $arguments = "";
if ($shell_call =~ /^(\/.*?) (.*)$/)
{
$program = $1;
$arguments = $2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => {
"s1:program" => $program,
"s2:arguments" => $arguments,
}});
}
if ($background)
{
# Prepend '/tmp/' to STDOUT and/or STDERR output files, if needed.
if (($stderr_file) && ($stderr_file !~ /^\//))
{
$stderr_file = "/tmp/".$stderr_file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { stderr_file => $stderr_file }});
}
if (($stdout_file) && ($stdout_file !~ /^\//))
{
$stdout_file = "/tmp/".$stdout_file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { stdout_file => $stdout_file }});
}
my $process = Proc::Simple->new();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { process => $process }});
# Setup output files
if (($stderr_file) && ($stdout_file))
{
$process->($stdout_file, $stderr_file);
}
elsif ($stdout_file)
{
$process->($stdout_file, undef);
}
elsif ($stderr_file)
{
$process->(undef, $stderr_file);
}
my $status = $process->start($shell_call);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { status => $status }});
# We'll return the handle instead of output.
$output = $process;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { output => $output }});
}
else
{
$output = "";
open (my $file_handle, $shell_call.$redirect." |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => $secure, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
while(<$file_handle>)
{
@ -192,6 +267,7 @@ sub call
$output =~ s/\n$//s;
}
}
}
return($output);
}

@ -64,15 +64,11 @@
#
# NOTE: If the archive directory doesn't exist, Anvil! will create it
# automatically the first time it is needed.
sys::database::archive::compress = 1
sys::database::archive::count = 50000
sys::database::archive::directory = /usr/local/anvil/archives/
sys::database::archive::division = 60000
sys::database::archive::trigger = 100000
# This is the schema for the Anvil! database.
sys::database::schema = /usr/share/anvil/anvil.sql
#sys::database::archive::compress = 1
#sys::database::archive::count = 50000
#sys::database::archive::directory = /usr/local/anvil/archives/
#sys::database::archive::division = 60000
#sys::database::archive::trigger = 100000
# This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is
# useful when doing very large transacions, like resync'ing a large table, by limiting how long a given

@ -24,13 +24,11 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
### NOTE: We'll print the headers only when we need to. If we print them hear, it will block cookies being set.
# Set the log level to 2. Setting 3 slows he program down a LOT.
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
# Read the config and then connect to the database.
$anvil->Storage->read_config();
@ -53,12 +51,12 @@ if (not -e $anvil->data->{path}{data}{host_uuid})
print_and_exit($anvil);
}
my $connections = $anvil->Database->connect({
$anvil->Database->connect({
sql_file => $anvil->data->{sys}{database}{schema},
test_table => "network_interfaces",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132", variables => { connections => $connections }});
if (not $connections)
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
if (not $anvil->data->{sys}{db_connections})
{
# No databases, exit.
print $anvil->Words->string({key => "error_0003"});
@ -575,6 +573,7 @@ sub add_sync_peer
if ($anvil->data->{cgi}{confirm}{value})
{
# OK, save!
}
else
{

@ -1,7 +1,7 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Striker</title>
<META HTTP-EQUIV="Refresh" CONTENT="0; URL=cgi-bin/home">
<META HTTP-EQUIV="Refresh" CONTENT="0; URL=cgi-bin/striker">
</head>
<body>
Hi, I'll be right with you...

@ -39,6 +39,7 @@ Requires: perl-JSON
Requires: perl-Log-Journald
Requires: perl-Net-SSH2
Requires: perl-NetAddr-IP
Requires: perl-Proc-Simple;
Requires: perl-Sys-Syslog
Requires: perl-Time-HiRes
Requires: perl-XML-Simple

@ -286,6 +286,8 @@ The database connection error was:
<key name="log_0185"><![CDATA[<unknown>]]></key> <!-- Used in some cases when a variable isn't known -->
<key name="log_0186"><![CDATA[<suppressed>]]></key> <!-- Used in some cases when a variable isn't known -->
<key name="log_0187">UUID cache file: [#!data!path::data::host_uuid!#] doesn't exists and we're not running as root. Unable to proceed.</key>
<key name="log_0188">Database archive check skipped, not running as root.</key>
<key name="log_0189">Database archiving is disabled, skipping archive checks.</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>
@ -419,6 +421,7 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st
<key name="striker_warning_0004">Accessing the peer over SSH worked, but a test connection to the database failed.</key>
<key name="striker_warning_0005">There was a problem reading the peer's UUID. Read: [#!variable!uuid!#], which appears to be invalid.</key>
<key name="striker_warning_0006">An SSH connection was established to: [#!variable!target!#], but we failed to establish a channel. The last error was: [#!variable!error!#].</key>
<key name="striker_warning_0007">The job: [#!variable!command!#] was picked up by: [#!variable!pid!#], but that process is not running and it appears to only be: [#!variable!percent!# %] complete. Restarting the job.</key>
<!-- Errors -->
<key name="error_0001">There are not enough network interfaces on this machine. You have: [#!variable!interface_count!#] interface(s), and you need at least: [#!variable!required_interfaces_for_single!#] interfaces to connect to the requested networks (one for Back-Channel and one for each Internet-Facing network).</key>

@ -29,9 +29,7 @@ $| = 1;
$< = $>;
$( = $);
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 0});
# Read switches
$anvil->Get->switches;
@ -49,9 +47,9 @@ if (($< != 0) && ($> != 0))
}
# Connect
my $connections = $anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { connections => $connections }});
if (not $connections)
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
if (not $anvil->data->{sys}{db_connections})
{
# No databases, exit.
print $anvil->Words->string({key => "error_0003"})."\n";

@ -26,9 +26,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1});
# Read switches
$anvil->data->{switches}{'no-reboot'} = 0;
@ -47,14 +45,10 @@ if (($< != 0) && ($> != 0))
}
# Connect
my $connections = $anvil->Database->connect({
sql_file => $anvil->data->{sys}{database}{schema},
test_table => "network_interfaces",
});
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0031"});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132", variables => { connections => $connections }});
if (not $connections)
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
if (not $anvil->data->{sys}{db_connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "error_0003"});

@ -13,6 +13,7 @@
use strict;
use warnings;
use Anvil::Tools;
use Proc::Simple;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
@ -24,8 +25,12 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 1});
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server.
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
# There are some things we only want to run on (re)start and don't need to always run.
run_once($anvil);
@ -33,6 +38,10 @@ run_once($anvil);
# Calculate my sum so that we can exit if it changes later.
$anvil->Storage->record_md5sums;
# Disconnect. We'll reconnect inside the loop
$anvil->Database->disconnect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
# These are the things we always want running.
while(1)
{
@ -62,7 +71,8 @@ $anvil->nice_exit({code => 0});
# Functions #
#############################################################################################################
# These are tools that don't need to constantly run.
# These are tools that don't need to constantly run. They'll typically run when the server starts up or the
# daemon is restarted or reloaded.
sub run_once
{
my ($anvil) = @_;
@ -84,12 +94,157 @@ sub keep_running
{
my ($anvil) = @_;
# Check for jobs that were running and now exited.
if (exists $anvil->data->{processes})
{
foreach my $job_uuid (%{$anvil->data->{jobs}{handles}})
{
# If it's not a handle, delete it.
my $running = $anvil->data->{jobs}{handles}{$job_uuid}->poll();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
"jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid},
running => $running,
}});
# If it's not running, update the table to clear the 'job_picked_up_by' column.
if (not $running)
{
my $exit_status = $anvil->data->{jobs}{handles}{$job_uuid}->exit_status();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { exit_status => $exit_status }});
# Free up memory
$anvil->data->{jobs}{handles}{$job_uuid}->cleanup();
clear_job($anvil, $job_uuid);
}
}
}
# Update hardware state files.
update_state_file($anvil);
# Run any pending jobs by calling 'anvil-jobs' with the 'job_uuid' as a background process.
run_jobs($anvil);
return(0);
}
# This clears the 'job_picked_up_by'.
sub clear_job
{
my ($anvil, $job_uuid) = @_;
# Check the local network and running services and tune the firewall as things change.
my $query = "
UPDATE
jobs
SET
job_picked_up_by = '0',
modified_date = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})."
WHERE
job_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)."
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
return(0);
}
# This will check for any jobs that aren't at 100%. For each found, if 'picked_up_by' is set, a check is made
# to see if the PID is still alive. If it isn't, or if 'picked_up_by' is not set, the appropriate tool is
# invoked to handle it.
sub run_jobs
{
my ($anvil) = @_;
# Get a list of pending or incomplete jobs.
my $query = "
SELECT
job_uuid,
job_command,
job_data,
job_picked_up_by,
job_picked_up_at,
job_updated,
job_progress
FROM
jobs
WHERE
job_progress != 100
AND
job_host_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->Get->host_uuid)."
LIMIT 1;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $job_uuid = $row->[0];
my $job_command = $row->[1];
my $job_data = defined $row->[2] ? $row->[2] : "";
my $job_picked_up_by = $row->[3];
my $job_picked_up_at = $row->[4];
my $job_updated = $row->[5];
my $job_progress = $row->[6];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_by => $job_picked_up_by,
job_picked_up_at => $job_picked_up_at,
job_updated => $job_updated,
job_progress => $job_progress,
}});
# See if the job was picked up by another running instance.
if ($job_picked_up_by)
{
# Check if the PID is still active.
$anvil->System->pids({ignore_me => 1});
### TODO: Add a check to verify the job isn't hung.
# Skip if this job is in progress.
next if exists $anvil->data->{pids}{$job_picked_up_by};
# The previous job is gone, but the job isn't finished. Start it again.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "striker_warning_0007", variables => {
command => $job_command,
pid => $job_picked_up_by,
percent => $job_progress,
}});
clear_job($anvil, $job_uuid);
}
# Start the job, appending '--job-uuid' to the command.
$anvil->data->{jobs}{handles}{$job_uuid} = $anvil->System->call({
debug => 2,
background => 1,
stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout",
stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr",
shell_call => $job_command." --job-uuid ".$job_uuid,
source => $THIS_FILE,
line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid} }});
# Record the PID
my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pid => $pid }});
my $query = "
UPDATE
jobs
SET
job_picked_up_by = ".$anvil->data->{sys}{use_db_fh}->quote($pid).",
modified_date = ".$anvil->data->{sys}{use_db_fh}->quote($anvil->data->{sys}{db_timestamp})."
WHERE
job_uuid = ".$anvil->data->{sys}{use_db_fh}->quote($job_uuid)."
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
return(0);
}

@ -23,8 +23,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
# I should have been passed a 'job_uuid'. Read in the details.

@ -26,9 +26,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 0});
my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 0});
# Read switches
$anvil->Get->switches;

@ -28,8 +28,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1});
scan($anvil);

@ -18,18 +18,13 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
my $anvil = Anvil::Tools->new({log_level => 1, log_secure => 1});
$anvil->Storage->read_config({file => "/etc/anvil/anvil.conf"});
my $connections = $anvil->Database->connect({
sql_file => $anvil->data->{sys}{database}{schema},
test_table => "network_interfaces",
});
$anvil->Database->connect;
print $THIS_FILE." ".__LINE__."; connections: [".$connections."]\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0132", variables => { connections => $connections }});
if (not $connections)
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132", variables => { "sys::db_connections" => $anvil->data->{sys}{db_connections} }});
if (not $anvil->data->{sys}{db_connections})
{
# No databases, exit.
print $anvil->Words->string({key => "error_0003"})."\n";

Loading…
Cancel
Save