seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Create tools/anvil-jobs and units/anvil-jobs.service, which is a new daemon that will handle jobs that can take some time to finish.

Browse Source 
* Created Storage->record_md5sums() and Storage->check_md5sums for use in daemons. These will record the md5sums of the program itself, all perl modules and the words file. When check_md5sums is called, it returns '1' if any sums have changed, which daemons can trigger on to exit (and systemd will restart them). Removed the basic md5sum check from anvil-daemon and switched to this.
* Fixed how 'fatalstobrowsers' is invoked so that it only applies to programs running in a browser.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 7 years ago
parent bb48c090a7
commit 9648e8ba43
8 changed files with 240 additions and 30 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      Anvil/Tools.pm
  2. 6
      Anvil/Tools/Get.pm
  3. 138
      Anvil/Tools/Storage.pm
  4. 37
      tools/anvil-daemon
  5. 58
      tools/anvil-jobs
  6. 5
      tools/words.xml
  7. 12
      units/anvil-jobs.service

10
Anvil/Tools.pm
Unescape View File

@ -16,6 +16,7 @@ use warnings;
use Scalar::Util qw(weaken isweak); use Scalar::Util qw(weaken isweak);
use Time::HiRes; use Time::HiRes;
use Data::Dumper; use Data::Dumper;
use CGI;
my $THIS_FILE = "Tools.pm"; my $THIS_FILE = "Tools.pm";
### Methods; ### Methods;
@ -290,11 +291,11 @@ sub environment
if ($_[0]) if ($_[0])
{ {
$anvil->{ENV_VALUES}{ENVIRONMENT} = shift; $anvil->{ENV_VALUES}{ENVIRONMENT} = shift;
# Load the CGI stuff if we're in a browser
if ($anvil->{ENV_VALUES}{ENVIRONMENT} eq "html") if ($anvil->{ENV_VALUES}{ENVIRONMENT} eq "html")
{ {
# Load the CGI stuff if we're in a browser CGI::Carp->import(qw(fatalsToBrowser));
use CGI;
use CGI::Carp qw(fatalsToBrowser);
} }
} }
@ -326,7 +327,7 @@ sub nice_exit
# Report the runtime. # Report the runtime.
my $end_time = Time::HiRes::time; my $end_time = Time::HiRes::time;
my $run_time = $end_time - $anvil->data->{ENV_VALUES}{START_TIME}; my $run_time = $end_time - $anvil->data->{ENV_VALUES}{START_TIME};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:ENV_VALUES::START_TIME' => $anvil->data->{ENV_VALUES}{START_TIME}, 's1:ENV_VALUES::START_TIME' => $anvil->data->{ENV_VALUES}{START_TIME},
's2:end_time' => $end_time, 's2:end_time' => $end_time,
's3:run_time' => $run_time, 's3:run_time' => $run_time,
@ -743,6 +744,7 @@ sub _set_paths
pgrep => "/usr/bin/pgrep", pgrep => "/usr/bin/pgrep",
psql => "/usr/bin/psql", psql => "/usr/bin/psql",
'postgresql-setup' => "/usr/bin/postgresql-setup", 'postgresql-setup' => "/usr/bin/postgresql-setup",
pwd => "/usr/bin/pwd",
su => "/usr/bin/su", su => "/usr/bin/su",
systemctl => "/usr/bin/systemctl", systemctl => "/usr/bin/systemctl",
touch => "/usr/bin/touch", touch => "/usr/bin/touch",

6
Anvil/Tools/Get.pm
Unescape View File

@ -443,7 +443,11 @@ sub md5sum
my $shell_call = $anvil->data->{path}{exe}{md5sum}." ".$file; my $shell_call = $anvil->data->{path}{exe}{md5sum}." ".$file;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
$sum = $anvil->System->call({shell_call => $shell_call}); my $return = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return }});
# split the sum off.
$sum = ($return =~ /^(.*?)\s+$file$/)[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { sum => $sum }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { sum => $sum }});
} }

138
Anvil/Tools/Storage.pm
Unescape View File

@ -14,12 +14,14 @@ my $THIS_FILE = "Storage.pm";
### Methods; ### Methods;
# change_mode # change_mode
# change_owner # change_owner
# check_md5sums
# copy_file # copy_file
# find # find
# make_directory # make_directory
# read_config # read_config
# read_file # read_file
# read_mode # read_mode
# record_md5sums
# search_directories # search_directories
# write_file # write_file
@ -247,6 +249,87 @@ sub change_owner
return($error); return($error);
} }
=head2 check_md5sums
This is one half of a tool to let daemons detect when something they use has changed on disk and restart if any changes are found.
This checks the md5sum of the calling application and all perl modules that are loaded and compares them against the sums seem earlier via C<< record_md5sums >>. If any sums don't match, C<< 1 >> is returned. If no changes were seen, C<< 0 >> is returned.
=cut
sub check_md5sums
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
# We'll set this if anything has changed.
my $exit = 0;
my $caller = $0;
# Have we changed?
$anvil->data->{md5sum}{$caller}{now} = $anvil->Get->md5sum({file => $0});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"md5sum::${caller}::start_time" => $anvil->data->{md5sum}{$caller}{start_time},
"md5sum::${caller}::now" => $anvil->data->{md5sum}{$caller}{now},
}});
if ($anvil->data->{md5sum}{$caller}{now} ne $anvil->data->{md5sum}{$caller}{start_time})
{
# Exit.
$exit = 1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0013", variables => { file => $0 }});
}
# What about our modules?
foreach my $module (sort {$a cmp $b} keys %INC)
{
my $module_file = $INC{$module};
my $module_sum = $anvil->Get->md5sum({file => $module_file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
module => $module,
module_file => $module_file,
module_sum => $module_sum,
}});
$anvil->data->{md5sum}{$module_file}{now} = $module_sum;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"md5sum::${module_file}::start_time" => $anvil->data->{md5sum}{$module_file}{start_time},
"md5sum::${module_file}::now" => $anvil->data->{md5sum}{$module_file}{now},
}});
if ($anvil->data->{md5sum}{$module_file}{start_time} ne $anvil->data->{md5sum}{$module_file}{now})
{
# Changed.
$exit = 1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0013", variables => { file => $module_file }});
}
}
# Record sums for word files.
foreach my $file (sort {$a cmp $b} keys %{$anvil->data->{words}})
{
my $words_sum = $anvil->Get->md5sum({file => $file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file => $file,
words_sum => $words_sum,
}});
$anvil->data->{md5sum}{$file}{now} = $words_sum;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"md5sum::${file}::start_time" => $anvil->data->{md5sum}{$file}{start_time},
"md5sum::${file}::now" => $anvil->data->{md5sum}{$file}{now},
}});
if ($anvil->data->{md5sum}{$file}{start_time} ne $anvil->data->{md5sum}{$file}{now})
{
$exit = 1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0013", variables => { file => $file }});
}
}
# Exit?
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'exit' => $exit }});
return($exit);
}
=head2 copy_file =head2 copy_file
This copies a file, with a few additional checks like creating the target directory if it doesn't exist, aborting if the file has already been backed up before, etc. This copies a file, with a few additional checks like creating the target directory if it doesn't exist, aborting if the file has already been backed up before, etc.
@ -775,6 +858,52 @@ sub read_mode
return($say_mode); return($say_mode);
} }
=head2 record_md5sums
This is one half of a tool to let daemons detect when something they use has changed on disk and restart if any changes are found.
This records the md5sum of the calling application and all perl modules that are loaded. The values stored here will be compared against C<< check_md5sums >> later.
=cut
sub record_md5sums
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $caller = $0;
$anvil->data->{md5sum}{$caller}{start_time} = $anvil->Get->md5sum({file => $0});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "md5sum::${caller}::start_time" => $anvil->data->{md5sum}{$caller}{start_time} }});
foreach my $module (sort {$a cmp $b} keys %INC)
{
my $module_file = $INC{$module};
my $module_sum = $anvil->Get->md5sum({file => $module_file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
module => $module,
module_file => $module_file,
module_sum => $module_sum,
}});
$anvil->data->{md5sum}{$module_file}{start_time} = $module_sum;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "md5sum::${module_file}::start_time" => $anvil->data->{md5sum}{$module_file}{start_time} }});
}
# Record sums for word files.
foreach my $file (sort {$a cmp $b} keys %{$anvil->data->{words}})
{
my $words_sum = $anvil->Get->md5sum({file => $file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
file => $file,
words_sum => $words_sum,
}});
$anvil->data->{md5sum}{$file}{start_time} = $words_sum;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "md5sum::${file}::start_time" => $anvil->data->{md5sum}{$file}{start_time} }});
}
return(0);
}
=head2 search_directories =head2 search_directories
This method returns an array reference of directories to search within for files and directories. This method returns an array reference of directories to search within for files and directories.
@ -843,9 +972,18 @@ sub search_directories
# Convert '.' to $ENV{PWD} # Convert '.' to $ENV{PWD}
if ($directory eq ".") if ($directory eq ".")
{
# When run from systemd, there is no PWD environment variable, so we'll do a system call.
if ($ENV{PWD})
{ {
$directory = $ENV{PWD}; $directory = $ENV{PWD};
} }
else
{
# pwd returns '/', which isn't helpful, so we'll skip this.
next;
}
}
# Skip duplicates # Skip duplicates
next if exists $seen_directories->{$directory}; next if exists $seen_directories->{$directory};

37
tools/anvil-daemon
Unescape View File

@ -7,8 +7,9 @@
# 0 = Normal exit # 0 = Normal exit
# 1 = md5sum of this program changed. Exited to reload. # 1 = md5sum of this program changed. Exited to reload.
# #
# TODO: At some point, we'll need to have a mechanism to fire off processes that might take a long time. # TODO:
# #
use strict; use strict;
use warnings; use warnings;
use Anvil::Tools; use Anvil::Tools;
@ -30,8 +31,7 @@ $anvil->Log->level({set => 2});
run_once($anvil); run_once($anvil);
# Calculate my sum so that we can exit if it changes later. # Calculate my sum so that we can exit if it changes later.
$anvil->data->{md5sum}{$THIS_FILE}{start_time} = $anvil->Get->md5sum({file => $0}); $anvil->Storage->record_md5sums;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "md5sum::${THIS_FILE}::start_time" => $anvil->data->{md5sum}{$THIS_FILE}{start_time} }});
# These are the things we always want running. # These are the things we always want running.
while(1) while(1)
@ -39,12 +39,20 @@ while(1)
# Loop and sleep for 2s. # Loop and sleep for 2s.
keep_running($anvil); keep_running($anvil);
# Exit if called with '--run-once' # Exit if called with '--run-once'
if ($anvil->data->{switches}{'run-once'}) if ($anvil->data->{switches}{'run-once'})
{ {
$anvil->nice_exit({code => 0}); $anvil->nice_exit({code => 0});
} }
# Has the file on disk changed?
if ($anvil->Storage->check_md5sums)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0014"});
$anvil->nice_exit({code => 1});
}
# Sleep now.
sleep 2; sleep 2;
} }
@ -61,11 +69,11 @@ sub run_once
# Check that the database is ready. # Check that the database is ready.
my $shell_call = $anvil->data->{path}{exe}{'anvil-prep-database'}; my $shell_call = $anvil->data->{path}{exe}{'anvil-prep-database'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my $database_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__}); my $database_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
if ($database_output) if ($database_output)
{ {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { database_output => $database_output }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }});
} }
return(0); return(0);
@ -79,19 +87,6 @@ sub keep_running
# Update hardware state files. # Update hardware state files.
update_state_file($anvil); update_state_file($anvil);
# Has the file on disk changed?
$anvil->data->{md5sum}{$THIS_FILE}{now} = $anvil->Get->md5sum({file => $0});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"md5sum::${THIS_FILE}::start_time" => $anvil->data->{md5sum}{$THIS_FILE}{start_time},
"md5sum::${THIS_FILE}::now" => $anvil->data->{md5sum}{$THIS_FILE}{now},
}});
if ($anvil->data->{md5sum}{$THIS_FILE}{now} ne $anvil->data->{md5sum}{$THIS_FILE}{start_time})
{
# Exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0013", variables => { file => $THIS_FILE }});
$anvil->nice_exit({code => 1});
}
return(0); return(0);
} }
@ -102,12 +97,12 @@ sub update_state_file
my ($anvil) = @_; my ($anvil) = @_;
my $shell_call = $anvil->data->{path}{exe}{'anvil-update-states'}; my $shell_call = $anvil->data->{path}{exe}{'anvil-update-states'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my $states_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__}); my $states_output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
if ($states_output) if ($states_output)
{ {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { states_output => $states_output }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { states_output => $states_output }});
} }
return(0); return(0);

58
tools/anvil-jobs
Unescape View File

@ -0,0 +1,58 @@
#!/usr/bin/perl
#
# This module handles running jobs that might take some time to complete.
#
# Exit codes;
# 0 = Normal exit
# 1 = md5sum of this program changed. Exited to reload.
#
# TODO:
#
use strict;
use warnings;
use Anvil::Tools;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
# Calculate my sum so that we can exit if it changes later.
$anvil->Storage->record_md5sums;
# These are the things we always want running.
while(1)
{
# Loop and sleep for 2s.
# TODO: DO THINGS HERE
# Exit if called with '--run-once'
if ($anvil->data->{switches}{'run-once'})
{
$anvil->nice_exit({code => 0});
}
# Has the file on disk changed?
if ($anvil->Storage->check_md5sums)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0014"});
$anvil->nice_exit({code => 1});
}
sleep 2;
}
$anvil->nice_exit({code => 0});
#############################################################################################################
# Functions #
#############################################################################################################

5
tools/words.xml
Unescape View File

@ -34,7 +34,8 @@ Author: Madison Kelly <mkelly@alteeve.ca>
<key name="message_0010">The SSH session to: [#!variable!target!#] was closed because 'no_cache' was set and there was an open SSH connection.</key> <key name="message_0010">The SSH session to: [#!variable!target!#] was closed because 'no_cache' was set and there was an open SSH connection.</key>
<key name="message_0011">Wrote the system UUID to the file: [#!variable!file!#] to enable the web based tools to read this system's UUID.</key> <key name="message_0011">Wrote the system UUID to the file: [#!variable!file!#] to enable the web based tools to read this system's UUID.</key>
<key name="message_0012">Wrote the journald config file: [#!variable!file!#] to disable rate limiting to ensure high log levels are not lost.</key> <key name="message_0012">Wrote the journald config file: [#!variable!file!#] to disable rate limiting to ensure high log levels are not lost.</key>
<key name="message_0013">The md5sum of: [#!variable!file!#] has changed since the daemon started. Exiting to reload.</key> <key name="message_0013">The md5sum of: [#!variable!file!#] has changed since the daemon started.</key>
<key name="message_0014">Exiting to reload.</key>
<!-- Log entries --> <!-- Log entries -->
<key name="log_0001">Starting: [#!variable!program!#].</key> <key name="log_0001">Starting: [#!variable!program!#].</key>
@ -204,7 +205,7 @@ The database connection error was:
<key name="log_0132">Connected to: [#!variable!connections!#] database(s).</key> <key name="log_0132">Connected to: [#!variable!connections!#] database(s).</key>
<key name="log_0133">Failed to read the system UUID. Received a non-UUID string: [#!variable!uuid!#]. Is the user: [#!variable!user!#] in the 'kmem' group?</key> <key name="log_0133">Failed to read the system UUID. Received a non-UUID string: [#!variable!uuid!#]. Is the user: [#!variable!user!#] in the 'kmem' group?</key>
<key name="log_0134">The read host UUID: [#!variable!uuid!#] does not appear to be a valid UUID.</key> <key name="log_0134">The read host UUID: [#!variable!uuid!#] does not appear to be a valid UUID.</key>
<key name="log_0135">Runtime was approximately: [#!variable!runtime!#].</key> <key name="log_0135">Runtime was approximately: [#!variable!runtime!#] seconds.</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. --> <!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key> <key name="t_0000">Test</key>

12
units/anvil-jobs.service
Unescape View File

@ -0,0 +1,12 @@
[Unit]
Description=Anvil! Intelligent Availability Platform - Daemon for handling jobs that take some time to complete
Wants=network.target
[Service]
Type=simple
ExecStart=/usr/sbin/anvil/anvil-jobs
ExecStop=/bin/kill -WINCH ${MAINPID}
Restart=always
[Install]
WantedBy=multi-user.target
Write Preview
Loading…
Cancel
Save