seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Local modifications to ClusterLabs/Anvil by Alteeve
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
364 Commits
2 Branches
0 Tags
21 MiB
 
 
 
 
 
 
Tag: Branch: Tree: b8816382b8
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'b8816382b8'
${ noResults }
anvil/tools/anvil-daemon

1206 lines
47 KiB
Raw Blame History

#!/usr/bin/perl
#
# This is the master daemon that manages all periodically run processes on Striker dashboards, Anvil! cluster
# nodes and DR hosts.
#
# Exit codes;
# 0 = Normal exit or md5sum of this program changed and it exited to reload.
# 1 =
# 2 = Unable to connect to any database, even after trying to initialize the local system.
#
# TODO:
# - Need to check what kind of machine this is and not prep the database unless its a dashboard.
# - Add a "running: pending,yes,done,dead" and show an appropriate icon beside jobs
# - Decide if holding before the main loop until 'systemctl is-system-running' returns 'running' is a good
# idea or not.
#
# NOTE:
# - For later; 'reboot --force --force' immediately kills the OS, like disabling ACPI on EL6 and hitting the
# power button. Might be useful in ScanCore down the road.
#
# Switches:
#
# --main-loop-only
#
# This skips the one-time, start-up tasks and just goes into the main-loop,
#
# --no-start
#
# This will prevent any pending jobs from being picked up and started in this run. Note that other job checks will still happen.
#
# --refresh-json
#
# This just updates the JSON files used by the web interface. It is the same as '--run-once --main-loop-only --no-start'
#
# --run-once
#
# This will tell the program to exit after runn the main loop once.
#
# --startup-only
#
# This will tell the program to exit after running the start up tasks, so the main loop won't run.
#
use strict;
use warnings;
use Anvil::Tools;
use Proc::Simple;
#use Time::HiRes qw ( time sleep );
use JSON;
use HTML::Strip;
use HTML::FromText;
use Data::Dumper;
use Text::Diff;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
# NOTE: Setting 'log_level' and 'log_secure' here will get overridden in the main lopp. Use the Log methods
# in the loop as well to override defaults in code.
my $anvil = Anvil::Tools->new();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server.
$anvil->Database->connect({debug => 3, check_if_configured => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
# If I have no databases, sleep for a second and then exit (systemd will restart us).
if (not $anvil->data->{sys}{database}{connections})
{
# Try to configure the local database, and then try to connect again.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0201"});
prep_database($anvil);
sleep 1;
# Try connecting again
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# Still nothing, sleep and exit.
print $anvil->Words->string({key => "error_0003"})."\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 2});
}
}
# Read switches
$anvil->data->{switches}{'refresh-json'} = "";
$anvil->data->{switches}{'run-once'} = 0;
$anvil->data->{switches}{'main-loop-only'} = 0;
$anvil->data->{switches}{'no-start'} = 0;
$anvil->data->{switches}{'startup-only'} = 0;
$anvil->Get->switches;
if ($anvil->data->{switches}{'refresh-json'})
{
$anvil->data->{switches}{'run-once'} = 1;
$anvil->data->{switches}{'main-loop-only'} = 1;
$anvil->data->{switches}{'no-start'} = 1;
}
# There are some things we only want to run on (re)start and don't need to always run.
run_once($anvil) if not $anvil->data->{switches}{'main-loop-only'};
# Calculate my sum so that we can exit if it changes later.
$anvil->Storage->record_md5sums;
# Disconnect. We'll reconnect inside the loop
$anvil->Database->disconnect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0203"});
# This will prevent restarting while jobs are running.
$anvil->data->{sys}{jobs_running} = 0;
# What time is it, Mr. Fox?
my $now_time = time;
# Once a minute, we'll check the md5sums and see if we should restart.
# Once a day, we'll refresh an Install Target's RPM repository (has no effect on non-Striker dashboards).
$anvil->data->{timing}{minute_checks} = 60;
$anvil->data->{timing}{repo_update_interval} = 86400;
$anvil->data->{timing}{next_minute_check} = $now_time - 1;
$anvil->data->{timing}{next_repo_check} = $now_time; # We want to run on daemon startup
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:timing::minute_checks" => $anvil->data->{timing}{minute_checks},
"s2:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval},
"s3:now_time" => $now_time,
"s4:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check},
"s5:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check},
}});
# When we periodically check if system files have changed, we'll also ask Database>connect() to check if it
# needs to be configured or updated. This is done periodically as it is expensive to run on every loop.
my $check_if_database_is_configured = 0;
# These are the things we always want running.
while(1)
{
# Reload defaults, re-read the config and then connect to the database(s)
$anvil->_set_paths();
$anvil->_set_defaults(); # This reset the log level
$anvil->Storage->read_config(); # This reset the log level also
$anvil->Get->switches; # Re-read to let switches override again.
$anvil->Words->read();
$anvil->Database->connect({check_if_configured => $check_if_database_is_configured});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
# Mark that we don't want to check the database now.
$check_if_database_is_configured = 0;
if ($anvil->data->{sys}{database}{connections})
{
# Run the normal tasks
keep_running($anvil);
}
else
{
# No databases available, we can't do anything this run.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "log_0202"});
}
# Handle periodic tasks
handle_periodic_tasks($anvil);
# Exit if 'run-once' selected.
if ($anvil->data->{switches}{'run-once'})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "message_0055"});
$anvil->nice_exit({code => 0});
}
# Disconnect from the database(s) and sleep now.
$anvil->Database->disconnect();
sleep(2);
}
$anvil->nice_exit({code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
# This handles running tasks that only run on some loops.
sub handle_periodic_tasks
{
my ($anvil) = @_;
my $now_time = time;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:now_time" => $now_time,
"s2:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check},
"s3:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check},
}});
# Time to run once per minute tasks.
if ($now_time >= $anvil->data->{timing}{next_minute_check})
{
# Check the firewall needs to be updated.
check_firewall($anvil);
# Check to see if the PXE environment needs to be updated.
check_install_target($anvil);
# Check that the users we care about have ssh public keys and they're recorded in ssh_keys.
check_ssh_keys($anvil);
# Check if the files on disk have changed. Even if it is time to check, don't if a job is
# running.
if ((not $anvil->data->{timing}{jobs_running}) && ($anvil->Storage->check_md5sums))
{
# NOTE: We exit with '0' to prevent systemctl from showing a scary red message.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "message_0014"});
$anvil->nice_exit({code => 0});
}
# Mark that we want to check the database config next time.
$check_if_database_is_configured = 1;
# Update the next check time.
$anvil->data->{timing}{next_minute_check} = $now_time + $anvil->data->{timing}{minute_checks};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:timing::minute_checks" => $anvil->data->{timing}{minute_checks},
"s2:timing::next_minute_check" => $anvil->data->{timing}{next_minute_check},
}});
# Make sure the shared directories exist.
foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}})
{
my $directory = $anvil->data->{path}{directories}{shared}{$target};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
target => $target,
directory => $directory,
}});
if (not -e $anvil->data->{path}{directories}{shared}{$target})
{
my $failed = $anvil->Storage->make_directory({
directory => $directory,
group => "apache",
user => "apache",
mode => "0775",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
if ($failed)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0254", variables => {
directory => $directory,
}});
}
else
{
# Success
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0255", variables => {
directory => $directory,
}});
}
}
}
}
### NOTE: We call it once/day, but this will also trigger on restart of anvil-daemon. As such, we
### don't use '--force' and let striker-manage-install-target skip the repo update if it happened
### recently enough.
# Is it time to refresh RPM packages on Install Target hosts?
if ($now_time >= $anvil->data->{timing}{next_repo_check})
{
# Record a job, don't call it directly. It takes too long to run.
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE,
line => __LINE__,
job_command => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --refresh",
job_data => "",
job_name => "install-target::refresh",
job_title => "job_0015",
job_description => "job_0017",
job_progress => 0,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }});
# Update the next check time.
$anvil->data->{timing}{next_repo_check} = $now_time + $anvil->data->{timing}{repo_update_interval};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval},
"s2:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check},
}});
}
return(0);
}
# Check that the host's fingerprint and users we care about have ssh public keys and they're recorded in ssh_keys.
sub check_ssh_keys
{
my ($anvil) = @_;
# Get a list of machine host keys and user public keys from other machines.
get_other_keys($anvil);
# Users to check:
# root, admin, hacluster
foreach my $user ("root", "admin")
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { user => $user }});
my $user_home = $anvil->Get->users_home({user => $user});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { user_home => $user_home }});
# If the user doesn't exist, their home directory won't either, so skip.
next if not $user_home;
next if not -d $user_home;
# If the user's ~/.ssh directory doesn't exist, we need to create it.
my $ssh_directory = $user_home."/.ssh";
$ssh_directory =~ s/\/\//\//g;
if (not -e $ssh_directory)
{
# Create it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0272", variables => { user => $user, directory => $ssh_directory }});
$anvil->Storage->make_directory({
directory => $ssh_directory,
user => $user,
group => $user,
mode => "0700",
});
if (not -e $ssh_directory)
{
# Failed ?
next;
}
}
my $ssh_private_key_file = $user_home."/.ssh/id_rsa";
my $ssh_public_key_file = $user_home."/.ssh/id_rsa.pub";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
ssh_public_key_file => $ssh_public_key_file,
ssh_private_key_file => $ssh_private_key_file,
}});
if (not -e $ssh_public_key_file)
{
# Generate the SSH keys.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0270", variables => { user => $user }});
my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'ssh-keygen'}." -t rsa -N \"\" -b 8191 -f ".$ssh_private_key_file});
if (-e $ssh_public_key_file)
{
# Success!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0271", variables => { user => $user, output => $output }});
}
else
{
# Failed?
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "error_0057", variables => { user => $user, output => $output }});
next;
}
}
# Now read in the key.
my $users_public_key = $anvil->Storage->read_file({file => $ssh_public_key_file});
$users_public_key =~ s/\n$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { users_public_key => $users_public_key }});
# Now store the key in the 'host_key' table, if needed.
my $host_key_uuid = $anvil->Database->insert_or_update_host_keys({
debug => 3,
host_key_host_uuid => $anvil->Get->host_uuid,
host_key_public_key => $users_public_key,
host_key_user_name => $user,
});
# Read in the existing 'known_hosts' file, if it exists. The 'old' and 'new' variables will
# be used when looking for needed changes.
my $known_hosts_file_body = "";
my $known_hosts_old_body = "";
my $known_hosts_new_body = "";
my $known_hosts_file = $ssh_directory."/known_hosts";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { known_hosts_file => $known_hosts_file }});
if (-e $known_hosts_file)
{
$known_hosts_file_body = $anvil->Storage->read_file({file => $known_hosts_file});
$known_hosts_old_body = $known_hosts_file_body;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { known_hosts_file_body => $known_hosts_file_body }});
}
# Read in the existing 'authorized_keys' file, if it exists.
my $authorized_keys_file_body = "";
my $authorized_keys_old_body = "";
my $authorized_keys_new_body = "";
my $authorized_keys_file = $ssh_directory."/authorized_keys";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { authorized_keys_file => $authorized_keys_file }});
if (-e $authorized_keys_file)
{
$authorized_keys_file_body = $anvil->Storage->read_file({file => $authorized_keys_file});
$authorized_keys_old_body = $authorized_keys_file_body;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { authorized_keys_file_body => $authorized_keys_file_body }});
}
### Walk through each host we now know of. As we we do, loop through the old file body to see
### if it exists. If it does, and the key has changed, update the line with the new key. If
### it isn't found, add it. Once we check the old body for this entry, change the "old" body
### to the new one, then repeat the process.
# Look at all the hosts I know about (other than myself) and see if any of the machine or
# user keys either don't exist or have changed.
my $update_known_hosts = 0;
my $update_authorized_keys = 0;
my $known_hosts_new_lines = "";
my $authorized_keys_new_lines = "";
# Check for changes to known_hosts
foreach my $host_uuid (keys %{$anvil->data->{peers}{ssh_keys}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_uuid => $host_uuid }});
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{peers}{ssh_keys}{$host_uuid}{host}})
{
my $key = $anvil->data->{peers}{ssh_keys}{$host_uuid}{host}{$host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:host_name' => $host_name,
's2:key' => $key,
}});
# Is this in the file and, if so, has it changed?
my $found = 0;
my $test_line = $host_name." ".$key;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_line => $test_line }});
foreach my $line (split/\n/, $known_hosts_old_body)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
if ($line eq $test_line)
{
# No change needed, key is the same.
$found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { found => $found }});
}
elsif ($line =~ /^$host_name /)
{
# Key has changed, update.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0274", variables => {
machine => $host_name,
old_key => $line,
new_key => $test_line,
}});
$found = 1;
$line = $test_line;
$update_known_hosts = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
found => $found,
line => $line,
update_known_hosts => $update_known_hosts,
}});
}
$known_hosts_new_body .= $line."\n";
}
# If we didn't find the key, add it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { found => $found }});
if (not $found)
{
$update_known_hosts = 1;
$known_hosts_new_lines .= $test_line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:update_known_hosts' => $update_known_hosts,
's2:known_hosts_new_lines' => $known_hosts_new_lines,
}});
}
# Move the new body over to the old body (even though it may not have
# changed) and then clear the new body to prepare for the next pass.
$known_hosts_old_body = $known_hosts_new_body;
$known_hosts_new_body = "";
}
}
# Lastly, copy the last version of the old body to the new body,
$known_hosts_new_body = $known_hosts_old_body.$known_hosts_new_lines;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:update_known_hosts' => $update_known_hosts,
's2:known_hosts_file_body' => $known_hosts_file_body,
's3:known_hosts_new_body' => $known_hosts_new_body,
's4:difference' => diff \$known_hosts_file_body, \$known_hosts_new_body, { STYLE => 'Unified' },
}});
# Check for changes to authorized_keys
foreach my $host_uuid (keys %{$anvil->data->{peers}{ssh_keys}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_uuid => $host_uuid }});
foreach my $host_key_user_name (sort {$a cmp $b} keys %{$anvil->data->{peers}{ssh_keys}{$host_uuid}{user}})
{
my $host_key_public_key = $anvil->data->{peers}{ssh_keys}{$host_uuid}{user}{$host_key_user_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:host_key_user_name' => $host_key_user_name,
's2:host_key_public_key' => $host_key_public_key,
}});
# The key in the file might have a different trailing suffix (user@hostname)
# and doesn't really matter. So we search by the key type and public key to
# see if it exists already.
my $found = 0;
my $test_line = ($host_key_public_key =~ /^(ssh-.*? .*?) /)[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { test_line => $test_line }});
foreach my $line (split/\n/, $authorized_keys_old_body)
{
# NOTE: Use '\Q...\E' so that the '+' characters in the key aren't
# evaluated as part of the regex.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
if ($line =~ /^\Q$test_line\E/)
{
# No change needed, key is the same.
$found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { found => $found }});
}
# We don't look for changes (yet). Might be worth looking for stale
# keys by ckecking of the host at the end matches an entry in the
# database and then verifying the keys haven't changed, but that's
# for another day.
$authorized_keys_new_body .= $line."\n";
}
# If we didn't find the key, add it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { found => $found }});
if (not $found)
{
$update_authorized_keys = 1;
$authorized_keys_new_lines .= $host_key_public_key."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:update_authorized_keys' => $update_authorized_keys,
's2:authorized_keys_new_lines' => $authorized_keys_new_lines,
}});
}
# Move the new body over to the old body (even though it may not have
# changed) and then clear the new body to prepare for the next pass.
$authorized_keys_old_body = $authorized_keys_new_body;
$authorized_keys_new_body = "";
}
}
# Lastly, copy the last version of the old body to the new body,
$authorized_keys_new_body = $authorized_keys_old_body.$authorized_keys_new_lines;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
's1:update_authorized_keys' => $update_authorized_keys,
's2:authorized_keys_file_body' => $authorized_keys_file_body,
's3:authorized_keys_new_body' => $authorized_keys_new_body,
's4:difference' => diff \$authorized_keys_file_body, \$authorized_keys_new_body, { STYLE => 'Unified' },
}});
# Update the known_hosts files, if needed.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update_known_hosts => $update_known_hosts }});
if ($update_known_hosts)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0273", variables => { user => $user, file => $known_hosts_file }});
if (-e $known_hosts_file)
{
my $backup_file = $anvil->Storage->backup({
debug => 3,
fatal => 1,
file => $known_hosts_file,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { backup_file => $backup_file }});
if (-e $backup_file)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0154", variables => { source_file => $known_hosts_file, target_file => $backup_file }});
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "error_0058", variables => { file => $known_hosts_file }});
}
}
my $failed = $anvil->Storage->write_file({
debug => 3,
overwrite => 1,
file => $known_hosts_file,
body => $known_hosts_new_body,
user => $user,
group => $user,
mode => "0644",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
}
# Update the authorized_keys files, if needed.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { update_authorized_keys => $update_authorized_keys }});
if ($update_authorized_keys)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0273", variables => { user => $user, file => $authorized_keys_file }});
if (-e $authorized_keys_file)
{
my $backup_file = $anvil->Storage->backup({
debug => 3,
fatal => 1,
file => $authorized_keys_file,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { backup_file => $backup_file }});
if (-e $backup_file)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0154", variables => { source_file => $authorized_keys_file, target_file => $backup_file }});
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "error_0058", variables => { file => $authorized_keys_file }});
}
}
my $failed = $anvil->Storage->write_file({
debug => 3,
overwrite => 1,
file => $authorized_keys_file,
body => $authorized_keys_new_body,
user => $user,
group => $user,
mode => "0644",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
}
}
return(0);
}
# Get a list of machine host keys and user public keys from other machines.
sub get_other_keys
{
my ($anvil) = @_;
delete $anvil->data->{peers}{ssh_keys};
# Get the machine keys for other hosts.
my $query = "
SELECT
host_uuid,
host_name,
host_key
FROM
hosts
WHERE
host_uuid != ".$anvil->Database->quote($anvil->Get->host_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $host_uuid = $row->[0];
my $host_name = $row->[1];
my $host_key = $row->[2];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_uuid => $host_uuid,
host_name => $host_name,
host_key => $host_key,
}});
$anvil->data->{peers}{ssh_keys}{$host_uuid}{host}{$host_name} = $host_key;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"peers::ssh_keys::${host_uuid}::host::${host_name}" => $anvil->data->{peers}{ssh_keys}{$host_uuid}{host}{$host_name},
}});
# If the host name is the long host name, create another entry with the short name.
if ($host_name =~ /^(.*?)\./)
{
my $short_hostname = $1;
$anvil->data->{peers}{ssh_keys}{$host_uuid}{host}{$short_hostname} = $host_key;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"peers::ssh_keys::${host_uuid}::host::${short_hostname}" => $anvil->data->{peers}{ssh_keys}{$host_uuid}{host}{$short_hostname},
}});
}
# Find any IP addresses for this host.
my $query = "SELECT ip_address_address FROM ip_addresses WHERE ip_address_host_uuid = ".$anvil->Database->quote($host_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $ip_address_address = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
ip_address_address => $ip_address_address,
}});
$anvil->data->{peers}{ssh_keys}{$host_uuid}{host}{$ip_address_address} = $host_key;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"peers::ssh_keys::${host_uuid}::host::${ip_address_address}" => $anvil->data->{peers}{ssh_keys}{$host_uuid}{host}{$ip_address_address},
}});
}
}
# Now read in the public key for other users on other machines.
$query = "
SELECT
host_key_host_uuid,
host_key_user_name,
host_key_public_key
FROM
host_keys
WHERE
host_key_host_uuid != ".$anvil->Database->quote($anvil->Get->host_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
$results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $host_key_host_uuid = $row->[0];
my $host_key_user_name = $row->[1];
my $host_key_public_key = $row->[2];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_key_host_uuid => $host_key_host_uuid,
host_key_user_name => $host_key_user_name,
host_key_public_key => $host_key_public_key,
}});
$anvil->data->{peers}{ssh_keys}{$host_key_host_uuid}{user}{$host_key_user_name} = $host_key_public_key;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"peers::ssh_keys::${host_key_host_uuid}::user::${host_key_user_name}" => $anvil->data->{peers}{ssh_keys}{$host_key_host_uuid}{user}{$host_key_user_name},
}});
}
return(0);
}
# This calls striker-manage-install-target to see if the dhcpd is running or not. If it is or isn't, the config
# variable 'install-target::enabled' is set/updated. On non-Striker hosts, this simply returns without doing
# anything.
sub check_install_target
{
my ($anvil) = @_;
my $system_type = $anvil->System->get_host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { system_type => $system_type }});
if ($system_type ne "dashboard")
{
# Not a dashboard, nothing to do.
return(0);
}
my $status = "unavailable";
my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --status --check --no-refresh"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output }});
foreach my $line (split/\n/, $output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
if ($line =~ /status=(\d)/)
{
my $digit = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { digit => $digit }});
if ($digit == 0)
{
$status = "disabled";
}
elsif ($digit == 1)
{
$status = "enabled";
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { status => $status }});
last;
}
}
# Record the status
$anvil->Database->insert_or_update_variables({
variable_name => "install-target::enabled",
variable_source_uuid => $anvil->Get->host_uuid,
variable_source_table => "hosts",
variable_value => $status,
variable_default => "unavailable",
variable_description => "striker_0110",
variable_section => "system",
});
return(0);
}
# These are tools that don't need to constantly run. They'll typically run when the server starts up or the
# daemon is restarted or reloaded.
sub run_once
{
my ($anvil) = @_;
# Check that the database is ready.
prep_database($anvil);
# Check to see if we need to do boot-time tasks. We only run these if we've just booted
boot_time_tasks($anvil);
# Check the ssh stuff.
check_ssh_keys($anvil);
if ($anvil->data->{switches}{'startup-only'})
{
$anvil->nice_exit({code => 0});
}
return(0);
}
# Configure/update the firewall.
sub check_firewall
{
my ($anvil) = @_;
# Don't call this if we're not configured yet.
my $configured = $anvil->System->check_if_configured({debug => 3});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { configured => $configured }});
# Check the firewall needs to be updated.
if ($configured)
{
my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{'anvil-manage-firewall'}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output => $output, return_code => $return_code }});
}
return(0);
}
# This handles tasks that need to run on boot (if any)
sub boot_time_tasks
{
my ($anvil) = @_;
# If the uptime is less than ten minutes, clear the reboot flag.
my $uptime = $anvil->System->get_uptime;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { uptime => $uptime }});
# Now find out if a reboot is listed as needed and when it was last changed.
my $reboot_needed = 0;
my $changed_seconds_ago = 0;
my $query = "
SELECT
variable_value,
(SELECT extract(epoch from now()) - extract(epoch from modified_date)) AS changed_seconds_ago
FROM
variables
WHERE
variable_source_table = 'hosts'
AND
variable_source_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
AND
variable_name = 'reboot::needed'
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
if ($count)
{
$reboot_needed = $results->[0]->[0];
$changed_seconds_ago = $results->[0]->[1];
$changed_seconds_ago =~ s/^(\d+)\..*$/$1/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
reboot_needed => $reboot_needed,
changed_seconds_ago => $changed_seconds_ago,
}});
}
# If a reboot is needed, see if the uptime is less than the time since the reboot needed flag was
# set. If the uptime is less, then the system rebooted since it was requested so clear it. h/t to
# Lisa Seelye (@thedoh) for this idea!
my $difference = ($changed_seconds_ago - $uptime);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:reboot_needed" => $reboot_needed,
"s2:changed_seconds_ago" => $changed_seconds_ago,
"s3:uptime" => $uptime,
"s4:difference" => $difference,
}});
if (($reboot_needed) && ($uptime < $changed_seconds_ago))
{
# Clear the reboot request.
$reboot_needed = $anvil->System->reboot_needed({set => 0});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { reboot_needed => $reboot_needed }});
# Check to see if there was a reboot job in progress. If so, finish it off.
my $job_uuid = $anvil->Job->get_job_uuid({program => "anvil-manage-power"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_uuid => $job_uuid }});
if ($job_uuid)
{
# Update the percentage to '100' and then clear the old PID.
my $date_time = $anvil->Get->date_and_time();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { date_time => $date_time }});
$anvil->Job->update_progress({
progress => 100,
message => "message_0064,!!date_and_time!".$date_time."!!",
job_uuid => $job_uuid,
picked_up_by => 0,
});
}
}
# Check the firewall needs to be updated.
check_firewall($anvil);
return(0);
}
# Configure the local database, if needed.
sub prep_database
{
my ($anvil) = @_;
my $shell_call = $anvil->data->{path}{exe}{'striker-prep-database'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my ($database_output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
if ($database_output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { database_output => $database_output }});
}
return(0);
}
# These are tools that need to keep running.
sub keep_running
{
my ($anvil) = @_;
# Check for jobs that were running and now exited.
if (exists $anvil->data->{processes})
{
foreach my $job_uuid (%{$anvil->data->{jobs}{handles}})
{
# If it's not a handle, delete it.
my $running = $anvil->data->{jobs}{handles}{$job_uuid}->poll();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid},
running => $running,
}});
# If it's not running, update the table to clear the 'job_picked_up_by' column.
if (not $running)
{
my $exit_status = $anvil->data->{jobs}{handles}{$job_uuid}->exit_status();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
job_uuid => $job_uuid,
exit_status => $exit_status,
}});
# Free up memory
$anvil->data->{jobs}{handles}{$job_uuid}->cleanup();
$anvil->Job->clear({job_uuid => $job_uuid});
}
}
}
# Update hardware state files if the system isn't configured. Running it always is too intensive.
my $configured = $anvil->System->check_if_configured;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { configured => $configured }});
if (not $configured)
{
update_state_file($anvil);
}
# Run any pending jobs by calling 'anvil-jobs' with the 'job_uuid' as a background process.
run_jobs($anvil);
return(0);
}
# This will check for any jobs that aren't at 100%. For each found, if 'picked_up_by' is set, a check is made
# to see if the PID is still alive. If it isn't, or if 'picked_up_by' is not set, the appropriate tool is
# invoked to handle it.
sub run_jobs
{
my ($anvil) = @_;
# This will be set to 1 if any jobs are not complete, preventing a restart of the daemon if it's
# changed on disk.
$anvil->data->{sys}{jobs_running} = 0;
# We'll also update the jobs.json file.
my $jobs_file = "{\"jobs\":[\n";
# Get a list of pending or incomplete jobs.
my $return = $anvil->Database->get_jobs({ended_within => 300});
my $count = @{$return};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
'return' => $return,
count => $count,
}});
foreach my $hash_ref (@{$return})
{
my $job_uuid = $hash_ref->{job_uuid};
my $job_command = $hash_ref->{job_command};
my $job_data = $hash_ref->{job_data};
my $job_picked_up_by = $hash_ref->{job_picked_up_by};
my $job_picked_up_at = $hash_ref->{job_picked_up_at};
my $job_updated = $hash_ref->{job_updated};
my $job_name = $hash_ref->{job_name};
my $job_progress = $hash_ref->{job_progress};
my $job_title = $hash_ref->{job_title};
my $job_description = $hash_ref->{job_description};
my $job_status = $hash_ref->{job_status};
my $started_seconds_ago = $job_picked_up_at ? (time - $job_picked_up_at) : 0;
my $updated_seconds_ago = $job_updated ? (time - $job_updated) : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_by => $job_picked_up_by,
job_picked_up_at => $job_picked_up_at,
job_updated => $job_updated,
job_name => $job_name,
job_progress => $job_progress,
job_title => $job_title,
job_description => $job_description,
job_status => $job_status,
started_seconds_ago => $started_seconds_ago,
updated_seconds_ago => $updated_seconds_ago,
}});
if ($job_progress ne "100")
{
$anvil->data->{sys}{jobs_running} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "sys::jobs_running" => $anvil->data->{sys}{jobs_running} }});
}
# See if the job was picked up by a now-dead instance.
if ($job_picked_up_by)
{
# Check if the PID is still active.
$anvil->System->pids({ignore_me => 1});
### TODO: Add a check to verify the job isn't hung.
# Skip if this job is in progress.
if (not exists $anvil->data->{pids}{$job_picked_up_by})
{
# If the job is done, just clear the 'job_picked_up_by' and be done.
if ($job_progress ne "100")
{
# It's possible that the job updated to 100% and exited after we
# gathered the job data, so we won't restart until we've seen it not
# running and not at 100% after 5 loops.
if ((not exists $anvil->data->{lost_job_count}{$job_uuid}) or (not defined $anvil->data->{lost_job_count}{$job_uuid}))
{
$anvil->data->{lost_job_count}{$job_uuid} = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "lost_job_count::${job_uuid}" => $anvil->data->{lost_job_count}{$job_uuid} }});
}
if ($anvil->data->{lost_job_count}{$job_uuid} > 5)
{
# The previous job is gone, but the job isn't finished. Start it again.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "striker_warning_0007", variables => {
command => $job_command,
pid => $job_picked_up_by,
percent => $job_progress,
}});
# Clear some variables.
$job_progress = 0;
$job_status = "message_0056";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
job_progress => $job_progress,
job_status => $job_status,
}});
# Clear the job.
$anvil->Job->clear({debug => 3, job_uuid => $job_uuid});
$anvil->data->{lost_job_count}{$job_uuid} = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "lost_job_count::${job_uuid}" => $anvil->data->{lost_job_count}{$job_uuid} }});
}
else
{
$anvil->data->{lost_job_count}{$job_uuid}++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "lost_job_count::${job_uuid}" => $anvil->data->{lost_job_count}{$job_uuid} }});
}
}
# Clear the PID
$job_picked_up_by = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { job_picked_up_by => $job_picked_up_by }});
}
}
# Convert the double-banged strings into a proper message.
my $say_title = $job_title ? $anvil->Words->parse_banged_string({key_string => $job_title}) : "";
my $say_description = $job_description ? $anvil->Words->parse_banged_string({key_string => $job_description}) : "";
my $say_status = $job_status ? $anvil->Words->parse_banged_string({key_string => $job_status}) : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
job_title => $job_title,
say_description => $say_description,
say_status => $say_status,
}});
# Make the status HTML friendly. Strip any embedded HTML then encode the text string.
if ($say_status)
{
my $html_strip = HTML::Strip->new();
$say_status = $html_strip->parse($say_status);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_status => $say_status }});
# Now make the resulting text string HTML friendly
my $text_to_html = HTML::FromText->new({
urls => 1,
email => 1,
lines => 1,
});
$say_status = $text_to_html->parse($say_status);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_status => $say_status }});
}
# Add this to the jobs.json file
my $json_string = to_json ({
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_at => $job_picked_up_at,
job_updated => $job_updated,
job_name => $job_name,
job_progress => $job_progress,
job_title => $say_title,
job_description => $say_description,
job_status => $say_status,
started_seconds_ago => $started_seconds_ago,
updated_seconds_ago => $updated_seconds_ago,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { json_string => $json_string }});
$jobs_file .= $json_string.",\n";
# If the job is done, move on.
next if $job_progress eq "100";
# If the job is not running, start it.
if ((not $job_picked_up_by) && ($job_progress ne "100") && (not $anvil->data->{switches}{'no-start'}))
{
# Start the job, appending '--job-uuid' to the command.
my $command = $job_command." --job-uuid ".$job_uuid;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0210", variables => { command => $command }});
($anvil->data->{jobs}{handles}{$job_uuid}, my $return_code) = $anvil->System->call({
debug => 3,
background => 1,
stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout",
stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr",
shell_call => $command,
source => $THIS_FILE,
line => __LINE__,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "jobs::handles::${job_uuid}" => $anvil->data->{jobs}{handles}{$job_uuid}, return_code => $return_code }});
# Log the PID (the job should update the database).
my $pid = $anvil->data->{jobs}{handles}{$job_uuid}->pid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { pid => $pid }});
}
}
# Close the jobs file.
$jobs_file =~ s/,\n$/\n/ms;
$jobs_file .= "]}\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { jobs_file => $jobs_file }});
# Write the JSON file
my $output_json = $anvil->data->{path}{directories}{html}."/status/jobs.json";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output_xml => $output_json }});
$anvil->Storage->write_file({
file => $output_json,
body => $jobs_file,
overwrite => 1,
mode => "0644",
user => "apache",
group => "apache"
});
return(0);
}
# This calls 'anvil-update-states' which will scan the local machine's state (hardware and software) and
# record write it out to an HTML file
sub update_state_file
{
my ($anvil) = @_;
my ($states_output, $return_code) = $anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{'anvil-update-states'}, source => $THIS_FILE, line => __LINE__});
if ($states_output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { states_output => $states_output }});
}
return(0);
}