* Renamed the 'ScanCore' executable to just 'scancore', moved it into the standard 'tools' directory and changed the agents directory to '/usr/sbin/scancore-agents'.
* Got scancore scanning the agents directory, and properly holding on startup until at least one database is available (instead of exiting), and holding on startup until the local system is configured. * Created the skeleton of the first scan agent; scan-network. * Fixed a bug in Storage->check_md5sums() where dynamically loaded modules, loaded after the initial md5sum calcs, would cause the calling daemon to exit (possibly on every invocation). * Created the scancore.README that will eventually be the main scan agent guide / API document. Signed-off-by: Digimer <digimer@alteeve.ca>main
Digimer
6 years ago
parent
1c70fceb70
commit
946fce018a
12 changed files with 453 additions and 132 deletions
@ -1,120 +0,0 @@ |
||||
#!/usr/bin/perl |
||||
# |
||||
# This is the main ScanCore program. It is started/killed/recovered by anvil-daemon. |
||||
# |
||||
# Examples; |
||||
# |
||||
# Exit codes; |
||||
# 0 = Normal exit. |
||||
# 1 = No database connections available. |
||||
# |
||||
# TODO: |
||||
# |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use Anvil::Tools; |
||||
use Data::Dumper; |
||||
|
||||
# Disable buffering |
||||
$| = 1; |
||||
|
||||
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
||||
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
||||
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
||||
{ |
||||
$running_directory =~ s/^\./$ENV{PWD}/; |
||||
} |
||||
|
||||
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); |
||||
|
||||
$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}}); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0115", variables => { program => $THIS_FILE }}); |
||||
|
||||
# Read switches |
||||
$anvil->data->{switches}{'run-once'} = ""; |
||||
$anvil->Get->switches; |
||||
|
||||
# Connect to DBs. |
||||
$anvil->Database->connect; |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0132"}); |
||||
if (not $anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# No databases, exit. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"}); |
||||
|
||||
# We sleep before exiting so that we don't get into a high-speed loop of systemd re-invoking us. |
||||
sleep 2; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
# Calculate my sum so that we can exit if it changes later. |
||||
$anvil->Storage->record_md5sums; |
||||
|
||||
# Disconnect. We'll reconnect inside the loop |
||||
$anvil->Database->disconnect(); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "log_0203"}); |
||||
|
||||
# The main loop |
||||
while(1) |
||||
{ |
||||
# Reload defaults, re-read the config and then connect to the database(s) |
||||
$anvil->_set_paths(); |
||||
$anvil->_set_defaults(); |
||||
$anvil->Storage->read_config({force_read => 1, file => $anvil->data->{path}{configs}{'anvil.conf'}}); |
||||
$anvil->Database->connect({check_if_configured => $check_if_database_is_configured}); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0132"}); |
||||
|
||||
# Mark that we don't want to check the database now. |
||||
$check_if_database_is_configured = 0; |
||||
|
||||
if ($anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# Run the normal tasks |
||||
call_agents($anvil); |
||||
|
||||
|
||||
} |
||||
else |
||||
{ |
||||
# No databases available, we can't do anything this run. Sleep for a couple of seconds and |
||||
# then try again. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "log_0202"}); |
||||
|
||||
my $db_retry_interval = 2; |
||||
if ((exists $anvil->data->{ScanCore}{timing}{db_retry_interval}) && ($anvil->data->{ScanCore}{timing}{db_retry_interval} =~ /^\d+$/)) |
||||
{ |
||||
$db_retry_interval = $anvil->data->{ScanCore}{timing}{db_retry_interval}; |
||||
} |
||||
sleep($db_retry_interval); |
||||
next; |
||||
} |
||||
|
||||
# Exit if 'run-once' selected. |
||||
if ($anvil->data->{switches}{'run-once'}) |
||||
{ |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0055"}); |
||||
$anvil->nice_exit({code => 0}); |
||||
} |
||||
|
||||
# Disconnect from the database(s) and sleep now. |
||||
$anvil->Database->disconnect(); |
||||
my $run_interval = 30; |
||||
if ((exists $anvil->data->{ScanCore}{timing}{run_interval}) && ($anvil->data->{ScanCore}{timing}{run_interval} =~ /^\d+$/)) |
||||
{ |
||||
$run_interval = $anvil->data->{ScanCore}{timing}{run_interval}; |
||||
} |
||||
sleep($run_interval); |
||||
} |
||||
|
||||
$anvil->nice_exit({code => 0}); |
||||
|
||||
|
||||
############################################################################################################# |
||||
# Functions # |
||||
############################################################################################################# |
||||
|
||||
# This invokes all scan agents found in 'path::directories::scan_agents' |
||||
sub call_agents |
||||
{ |
||||
} |
||||
@ -0,0 +1,38 @@ |
||||
#!/usr/bin/perl |
||||
# |
||||
# This scans the local network (interfaces, bonds and bridges). |
||||
# |
||||
# Examples; |
||||
# |
||||
# Exit codes; |
||||
# 0 = Normal exit. |
||||
# 1 = No database connections available. |
||||
# |
||||
# TODO: |
||||
# - Decide if it's worth having a separate ScanCore.log file or just feed into anvil.log. |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use Anvil::Tools; |
||||
use Data::Dumper; |
||||
|
||||
# Disable buffering |
||||
$| = 1; |
||||
|
||||
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
||||
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
||||
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
||||
{ |
||||
$running_directory =~ s/^\./$ENV{PWD}/; |
||||
} |
||||
|
||||
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); |
||||
|
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0115", variables => { program => $THIS_FILE }}); |
||||
|
||||
|
||||
$anvil->nice_exit({exit_code => 0}); |
||||
|
||||
############################################################################################################# |
||||
# Functions # |
||||
############################################################################################################# |
||||
@ -0,0 +1,20 @@ |
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
||||
<!-- |
||||
Company: Alteeve's Niche, Inc. |
||||
License: GPL v2+ |
||||
Author: Madison Kelly <mkelly@alteeve.ca> |
||||
|
||||
NOTE: All string keys MUST be prefixed with the agent name! ie: 'scan_network_log_0001'. |
||||
--> |
||||
|
||||
<words> |
||||
<meta version="3.0.0" languages="en_CA,jp"/> |
||||
<!-- Canadian English --> |
||||
<language name="en_CA" long_name="Canadian English" description="ScanCore scan agent that monitors local network interfaces, bonds and bridges."> |
||||
|
||||
<!-- Log entries --> |
||||
<key name="scan_network_log_0001">Starting: [#!variable!program!#].</key> |
||||
|
||||
</language> |
||||
</words> |
||||
@ -0,0 +1,257 @@ |
||||
#!/usr/bin/perl |
||||
# |
||||
# This is the main ScanCore program. It is started/killed/recovered by anvil-daemon. |
||||
# |
||||
# Examples; |
||||
# |
||||
# Exit codes; |
||||
# 0 = Normal exit. |
||||
# 1 = No database connections available. |
||||
# |
||||
# TODO: |
||||
# - Decide if it's worth having a separate ScanCore.log file or just feed into anvil.log. |
||||
# |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use Anvil::Tools; |
||||
use Data::Dumper; |
||||
|
||||
# Disable buffering |
||||
$| = 1; |
||||
|
||||
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
||||
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
||||
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
||||
{ |
||||
$running_directory =~ s/^\./$ENV{PWD}/; |
||||
} |
||||
|
||||
my $anvil = Anvil::Tools->new({log_level => 2, log_secure => 1}); |
||||
|
||||
$anvil->Storage->read_config(); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0115", variables => { program => $THIS_FILE }}); |
||||
|
||||
# Read switches |
||||
$anvil->data->{switches}{'run-once'} = ""; |
||||
$anvil->Get->switches; |
||||
|
||||
# Calculate my sum so that we can exit if it changes later. |
||||
$anvil->Storage->record_md5sums; |
||||
|
||||
# Connect to DBs. |
||||
wait_for_database($anvil); |
||||
|
||||
# If we're not configured, sleep. |
||||
wait_until_configured($anvil); |
||||
|
||||
# Disconnect. We'll reconnect inside the loop |
||||
$anvil->Database->disconnect(); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0203"}); |
||||
|
||||
# The main loop |
||||
while(1) |
||||
{ |
||||
# Reload defaults, re-read the config and then connect to the database(s) |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0248"}); |
||||
$anvil->_set_paths(); |
||||
$anvil->_set_defaults(); |
||||
$anvil->Storage->read_config(); |
||||
$anvil->Database->connect(); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0132"}); |
||||
|
||||
if ($anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# Run the normal tasks |
||||
call_agents($anvil); |
||||
} |
||||
else |
||||
{ |
||||
# No databases available, we can't do anything this run. Sleep for a couple of seconds and |
||||
# then try again. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "log_0202"}); |
||||
|
||||
my $db_retry_interval = 2; |
||||
if ((exists $anvil->data->{scancore}{timing}{db_retry_interval}) && ($anvil->data->{scancore}{timing}{db_retry_interval} =~ /^\d+$/)) |
||||
{ |
||||
$db_retry_interval = $anvil->data->{scancore}{timing}{db_retry_interval}; |
||||
} |
||||
sleep($db_retry_interval); |
||||
next; |
||||
} |
||||
|
||||
# Exit if 'run-once' selected. |
||||
if ($anvil->data->{switches}{'run-once'}) |
||||
{ |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "message_0055"}); |
||||
$anvil->nice_exit({code => 0}); |
||||
} |
||||
|
||||
# Disconnect from the database(s) and sleep now. |
||||
$anvil->Database->disconnect(); |
||||
my $run_interval = 30; |
||||
if ((exists $anvil->data->{scancore}{timing}{run_interval}) && ($anvil->data->{scancore}{timing}{run_interval} =~ /^\d+$/)) |
||||
{ |
||||
$run_interval = $anvil->data->{scancore}{timing}{run_interval}; |
||||
} |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0249", variables => { run_interval => $run_interval }}); |
||||
sleep($run_interval); |
||||
|
||||
# In case something has changed, exit. |
||||
exit_if_sums_changed($anvil); |
||||
} |
||||
|
||||
$anvil->nice_exit({code => 0}); |
||||
|
||||
|
||||
############################################################################################################# |
||||
# Functions # |
||||
############################################################################################################# |
||||
|
||||
# This invokes all scan agents found in 'path::directories::scan_agents' |
||||
sub call_agents |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
# Get the list of scan agents on this system. |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
"path::directories::scan_agents" => $anvil->data->{path}{directories}{scan_agents}, |
||||
}}); |
||||
scan_directory($anvil, $anvil->data->{path}{directories}{scan_agents}); |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
# This checks to see if any files on disk have changed and, if so, exits. |
||||
sub exit_if_sums_changed |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
if ($anvil->Storage->check_md5sums) |
||||
{ |
||||
# NOTE: We exit with '0' to prevent systemctl from showing a scary red message. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "warn", key => "message_0014"}); |
||||
$anvil->nice_exit({code => 0}); |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
# This looks in the passed-in directory for scan agents or sub-directories (which will in turn be scanned). |
||||
sub scan_directory |
||||
{ |
||||
my ($anvil, $directory) = @_; |
||||
|
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }}); |
||||
local(*DIRECTORY); |
||||
opendir(DIRECTORY, $directory); |
||||
while(my $file = readdir(DIRECTORY)) |
||||
{ |
||||
next if $file eq "."; |
||||
next if $file eq ".."; |
||||
my $full_path = $directory."/".$file; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
file => $file, |
||||
full_path => $full_path, |
||||
}}); |
||||
# If we're looking at a directory, scan it. Otherwise, see if it's an executable and that it |
||||
# starts with 'scan-*'. |
||||
if (-d $full_path) |
||||
{ |
||||
# This is a directory, dive into it. |
||||
scan_directory($anvil, $full_path); |
||||
} |
||||
elsif (-x $full_path) |
||||
{ |
||||
# Now I only want to know if the file starts with 'scan-' |
||||
next if $file !~ /^scan-/; |
||||
|
||||
# If I am still alive, I am looking at a scan agent! |
||||
$anvil->data->{scancore}{agent}{$file} = $full_path; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
"scancore::agent::${file}" => $anvil->data->{scancore}{agent}{$file}, |
||||
}}); |
||||
} |
||||
} |
||||
closedir(DIRECTORY); |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
# This loops until it can connect to at least one database. |
||||
sub wait_for_database |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
$anvil->Database->connect; |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); |
||||
if (not $anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# No databases, sleep until one comes online. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0244"}); |
||||
|
||||
until($anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# Disconnect, Sleep, then check again. |
||||
$anvil->Database->disconnect(); |
||||
sleep 60; |
||||
|
||||
$anvil->_set_paths(); |
||||
$anvil->_set_defaults(); |
||||
$anvil->Storage->read_config(); |
||||
$anvil->Database->connect; |
||||
if ($anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# We're good |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0245"}); |
||||
} |
||||
else |
||||
{ |
||||
# Not yet... |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0244"}); |
||||
} |
||||
|
||||
# In case something has changed, exit. |
||||
exit_if_sums_changed($anvil); |
||||
} |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
# Wait until the local system has been configured. |
||||
sub wait_until_configured |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
my $configured = $anvil->System->check_if_configured; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { configured => $configured }}); |
||||
if (not $configured) |
||||
{ |
||||
# Sleep for a minute and check again. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0246"}); |
||||
until ($configured) |
||||
{ |
||||
# Sleep, then check. |
||||
sleep 60; |
||||
|
||||
$configured = $anvil->System->check_if_configured; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configured => $configured }}); |
||||
if ($configured) |
||||
{ |
||||
# We're good |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0247"}); |
||||
} |
||||
else |
||||
{ |
||||
# Not yet... |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "log_0246"}); |
||||
} |
||||
|
||||
# In case something has changed, exit. |
||||
exit_if_sums_changed($anvil); |
||||
} |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
@ -0,0 +1,67 @@ |
||||
ScanCore notes; |
||||
|
||||
= ScanCore = |
||||
|
||||
ScanCore runs as a daemon, periodically scanning for "Scan Agents" and invoking all agents it finds in (and |
||||
under) 'path::directories::scan_agents'. See 'Agents' below for details on writing new agents. |
||||
|
||||
If the local system is not configured, or if none of the databases are available, ScanCore will go into a |
||||
loop, sleeping for a period of time and then re-checking to see if the system is not configured or if at |
||||
least one database is available. Once read, it will serially execute all scan agents it finds. |
||||
|
||||
Each agent is given a period of time it is allowed to run for before it is terminated. This is controlled by |
||||
'scancore::timing::agent_runtime', but can be overridden on a per-agent basis with |
||||
'scancore::agent::<agent_name>::agent_runtime'. |
||||
|
||||
NOTE: It is strongly recommended to keep the average runtime of an agent as low as possible! |
||||
|
||||
To prevent putting too high a load on the host system, agents are called sequentially. So an agent that takes |
||||
a long time to run will cause all other agents to be delayed, and slow down how often post-scan checks can be |
||||
performed. |
||||
|
||||
= Agents = |
||||
|
||||
ScanCore Agents are self-contained executables that can be written in any language the user chooses. A |
||||
typical agent contains three files under a dedicated directory, itself under |
||||
'path::directories::scan_agents'. For example, the agent 'scan-network'; |
||||
|
||||
* /usr/sbin/scancore-agents/scan-network/scan-network - Main program |
||||
* /usr/sbin/scancore-agents/scan-network/scan-network.sql - SQL schema |
||||
* /usr/sbin/scancore-agents/scan-network/scan-network.xml - XML 'words' |
||||
|
||||
== Permissions == |
||||
|
||||
Given most agents are interacting with core systems, all agents are called with root priviledges. If your |
||||
agent doesn't need priviledged access, it is recommended that you drop to an unpriviledged user. |
||||
|
||||
If you provide your agent via an external RPM (or other mechanism), be sure to properly setup SELinux. It is |
||||
enabled and enforcing on production systems! |
||||
|
||||
== Naming == |
||||
|
||||
All scan agents *must* start with the name 'scan-X'. When ScanCore walks the agents directory, any file that |
||||
does not start with this name is ignored. |
||||
|
||||
== Main Program == |
||||
|
||||
This is the executable invoked by ScanCore. It should do a single scan and then exit. Keeping the total |
||||
runtime as short as possible should be a high priority! |
||||
|
||||
== SQL Schema == |
||||
|
||||
Most agents will want to store data in the ScanCore database (usually a postgres database called 'anvil', see |
||||
'database::X' entries in anvil.conf). If your tables are not found in a given database, this schema will be |
||||
loaded. |
||||
|
||||
At this time, there are Perl libraries (see 'perldoc Anvil::Tools::Database') that dramatically simplify |
||||
connecting to any available databases, handling resync when a given database falls behind, etc. If you plan |
||||
to write a scan agent in another language, porting these tools would be very much appreciated. Let us know |
||||
and we will be happy to assist however we can. |
||||
|
||||
== XML 'words' == |
||||
|
||||
Any strings used for logging or sending "alerts" to notification recipients are found in this file. Please |
||||
see 'words.xml' for more information on how this file is structured. |
||||
|
||||
NOTE: To avoid namespace collisions, it is STRONGLY recommended that all string keys start with your agent |
||||
name! Ie: 'scan_network_X'. |
||||
Loading…
Reference in new issue