From b5dc83d39c3d66b96afbde664592330218e62854 Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 6 Dec 2018 04:14:58 -0500 Subject: [PATCH] * Renamed anvil-configure-striker -> striker-configure-host and anvil-manage-install-target -> striker-manage-install-target as they're both Striker-specific. * Fixed a bug in Words->parse_banged_string where some variable strings were not being cleared, causing infinite loops. * Added job progress reporting in striker-manage-install-target, and made it only refresh the RPM repo when '--refresh' is specified (with --force now forcing the issue). This was done to allow adding it into anvil-daemon in such a way that it would only update the RPM repo once a day. Signed-off-by: Digimer --- Anvil/Tools.pm | 4 +- Anvil/Tools/Job.pm | 2 + Anvil/Tools/Storage.pm | 2 +- Anvil/Tools/Words.pm | 9 +- cgi-bin/striker | 4 +- html/skins/alteeve/pxe.txt | 1 - share/words.xml | 16 +- tools/anvil-daemon | 119 +++++++---- ...nfigure-striker => striker-configure-host} | 0 ...l-target => striker-manage-install-target} | 185 ++++++++++++------ 10 files changed, 228 insertions(+), 114 deletions(-) rename tools/{anvil-configure-striker => striker-configure-host} (100%) rename tools/{anvil-manage-install-target => striker-manage-install-target} (93%) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index a393afa2..1775876e 100755 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -933,12 +933,10 @@ sub _set_paths }, exe => { 'anvil-change-password' => "/usr/sbin/anvil-change-password", - 'anvil-configure-striker' => "/usr/sbin/anvil-configure-striker", 'anvil-daemon' => "/usr/sbin/anvil-daemon", 'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode", 'anvil-manage-power' => "/usr/sbin/anvil-manage-power", 'anvil-manage-striker-peers' => "/usr/sbin/anvil-manage-striker-peers", - 'anvil-manage-install-target' => "/usr/sbin/anvil-manage-install-target", 'anvil-prep-database' => "/usr/sbin/anvil-prep-database", 'anvil-report-memory' => "/usr/sbin/anvil-report-memory", 'anvil-update-states' => "/usr/sbin/anvil-update-states", @@ -983,6 +981,8 @@ sub _set_paths 'shutdown' => "/usr/sbin/shutdown", 'ssh-keyscan' => "/usr/bin/ssh-keyscan", strings => "/usr/bin/strings", + 'striker-configure-host' => "/usr/sbin/striker-configure-host", + 'striker-manage-install-target' => "/usr/sbin/striker-manage-install-target", stty => "/usr/bin/stty", su => "/usr/bin/su", systemctl => "/usr/bin/systemctl", diff --git a/Anvil/Tools/Job.pm b/Anvil/Tools/Job.pm index eedda4e3..630fd363 100755 --- a/Anvil/Tools/Job.pm +++ b/Anvil/Tools/Job.pm @@ -210,6 +210,8 @@ This is the UUID of the job to update. If it isn't set, but C<< jobs::job_uuid > If set, this message will be appended to C<< job_status >>. If set to 'C<< clear >>', previous records will be removed. +NOTE: This is in the format C<< [,!!![,...,!!!!!]] >>. Example; C<< foo_0001 >> or C<< foo_0002,!!bar!baz!! >>. + =head3 picked_up_by (optional, default '$$' (caller's PID)) If set, this is used for the C<< job_picked_up_by >> column. If it isn't set, the process ID of the caller is used. diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 7cd74e63..016e6f8c 100755 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -2094,7 +2094,7 @@ sub update_file else { # Update not needed. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0231", variables => { file => $file }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0231", variables => { file => $file }}); return(2); } diff --git a/Anvil/Tools/Words.pm b/Anvil/Tools/Words.pm index 49a7c785..5bb788fe 100755 --- a/Anvil/Tools/Words.pm +++ b/Anvil/Tools/Words.pm @@ -309,16 +309,15 @@ sub parse_banged_string while ($variable_string) { my $pair = ($variable_string =~ /^(!!.*?!.*?!!).*$/)[0]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { pair => $pair }}); - my ($variable, $value) = ($pair =~ /^!!(.*?)!(.*?)!!$/); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - variable => $variable, - value => $value, + "s1:pair" => $pair, + "s2:variable" => $variable, + "s3:value" => $value, }}); # Remove this pair - $variable_string =~ s/^$pair//; + $variable_string =~ s/^\Q$pair//; $variable_string =~ s/^,//; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { variable_string => $variable_string }}); diff --git a/cgi-bin/striker b/cgi-bin/striker index 355b7850..76558366 100755 --- a/cgi-bin/striker +++ b/cgi-bin/striker @@ -357,7 +357,7 @@ sub process_install_target my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ file => $THIS_FILE, line => __LINE__, - job_command => $anvil->data->{path}{exe}{'anvil-manage-install-target'}." --".$anvil->data->{cgi}{subtask}{value}, + job_command => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --".$anvil->data->{cgi}{subtask}{value}, job_data => "", job_name => "install-target::".$anvil->data->{cgi}{task}{value}, job_title => "job_0015", @@ -1033,7 +1033,7 @@ sub configure_striker debug => 3, file => $THIS_FILE, line => __LINE__, - job_command => $anvil->data->{path}{exe}{'anvil-configure-striker'}, + job_command => $anvil->data->{path}{exe}{'striker-configure-host'}, job_data => "form::config_step2", job_name => "configure::network", job_title => "job_0001", diff --git a/html/skins/alteeve/pxe.txt b/html/skins/alteeve/pxe.txt index 77bacce4..9a65c94c 100644 --- a/html/skins/alteeve/pxe.txt +++ b/html/skins/alteeve/pxe.txt @@ -27,7 +27,6 @@ subnet #!variable!network!# netmask 255.255.0.0 { ### Alteeve's Niche! Inc. - Anvil! Intelligent Availability(tm) Platform # License: GPLv2 -# Built: #!variable!date!# # Target: Network Install (PXE) # OS: #!variable!os!# # Machine: #!variable!say_type!# diff --git a/share/words.xml b/share/words.xml index a1d72077..c3b925dc 100644 --- a/share/words.xml +++ b/share/words.xml @@ -217,7 +217,7 @@ NOTE: Please be patient! Anvil! Node Dsaster Recovery (DR) Host Regenerating the source repository metadata. - [ Error ] - The comps.xml file: [#!variable!comps_xml!#] was not found. This provides package group information required for Install Target guests. Is 'anvil-striker-extra' installed? + [ Error ] - The comps.xml file: [#!variable!comps_xml!#] was not found. This provides package group information required for Install Target guests. Is the 'anvil-striker-extra' package installed? About to try to download aproximately: [#!variable!packages!#] packages needed to: - [#!variable!directory!#]. @@ -230,6 +230,8 @@ About to try to download aproximately: [#!variable!packages!#] packages needed t The attempt to disabled the 'Install Target' function failed! Please check the logs for details. The 'Install Target' function has been enabled. The attempt to enable the 'Install Target' function failed! Please check the logs for details. + [ Error ] - The comps.xml file: [#!variable!comps_xml!#] was found, but something failed when we tried to copy it to: [#!variable!target!#]. + Updated repository data. Starting: [#!variable!program!#]. @@ -504,8 +506,13 @@ The body of the file: [#!variable!file!#] does not match the new body. The file The file: [#!variable!file!#] will now be updated. There was a problem updating file: [#!variable!file!#], expected the write to return '0' but got: [#!variable!return!#]. Please check the logs for details. Failed to backup the file: [#!variable!source!#] to: [#!variable!destination!#]. Details may be found in the logs above. - #!free!# - Skipping the RPM repository refresh. The next scheduled refresh will be done in: [#!variable!next_refresh!#] second(s). Use '#!variable!program!# --refresh' to force an immediate refresh. + Refreshing RPM repository has been disabled in [#!data!path::configs::anvil.conf!#] ('install-manifest::refresh-packages' is set). Not refreshing. + Skipping the RPM repository refresh. The next scheduled refresh will be done in: [#!variable!next_refresh!#] second(s). Override with '--force'. + RPM repository refresh required, [#!data!path::directories::packages!#] doesn't exist (likely this is the first run or the directory was deleted). + RPM repository refresh required, it has been more than: [#!variable!seconds!#] seconds since the last refresh (or no previous refresh was logged). + 'Install Target' job: [#!data!switches::job-uuid!#] picked up. + 'Install Target' job: [#!data!switches::job-uuid!#] aborted, system not yet configured. + Package list loaded. Test @@ -529,7 +536,7 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st Anvil! Striker ScanCore - Alteeve's Niche! Inc., Toronto, Ontario, Canada]]> + Alteeve's Niche! Inc., Toronto, Ontario, Canada]]> Anvil!]]> Current Network Interfaces and States @@ -675,6 +682,7 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st The Striker peer will now be removed from the local configuration. Manager Install Target. Enable or disable the 'Install Target' feature. + Update the 'Install Target' source files and refresh RPM repository. The IP address will change. You will need to reconnect after applying these changes. diff --git a/tools/anvil-daemon b/tools/anvil-daemon index e2d3e4c1..24683299 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -95,14 +95,21 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure = # This will prevent restarting while jobs are running. $anvil->data->{sys}{jobs_running} = 0; -# Once a minute, we'll check the md5sums and see if we should restart. We don't check every loop as it places a -my $system_check = 60; -my $now_time = time; -my $next_check = $now_time + $system_check; -$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "s1:system_check" => $system_check, - "s2:now_time" => $now_time, - "s3:next_check" => $next_check, +# What time is it, Mr. Fox? +my $now_time = time; + +# Once a minute, we'll check the md5sums and see if we should restart. +# Once a day, we'll refresh an Install Target's RPM repository (has no effect on non-Striker dashboards). +$anvil->data->{timing}{sum_check_interval} = 60; +$anvil->data->{timing}{repo_update_interval} = 86400; +$anvil->data->{timing}{next_sum_check} = $now_time + $anvil->data->{timing}{sum_check_interval}; +$anvil->data->{timing}{next_repo_check} = $now_time; # We want to run on daemon startup +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { + "s1:timing::sum_check_interval" => $anvil->data->{timing}{sum_check_interval}, + "s2:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval}, + "s3:now_time" => $now_time, + "s4:timing::next_sum_check" => $anvil->data->{timing}{next_sum_check}, + "s5:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check}, }}); # When we periodically check if system files have changed, we'll also ask Database>connect() to check if it @@ -133,16 +140,45 @@ while(1) $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "log_0202"}); } - # Has the file on disk changed? - $now_time = time; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "s1:now_time" => $now_time, - "s2:next_check" => $next_check, + # Handle periodic tasks + handle_periodic_tasks($anvil); + + # Exit if 'run-once' selected. + if ($anvil->data->{switches}{'run-once'}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0055"}); + $anvil->nice_exit({code => 0}); + } + + # Disconnect from the database(s) and sleep now. + $anvil->Database->disconnect(); + sleep(1); +} + +$anvil->nice_exit({code => 0}); + + +############################################################################################################# +# Functions # +############################################################################################################# + +# This handles running tasks that only run on some loops. +sub handle_periodic_tasks +{ + my ($anvil) = @_; + + my $now_time = time; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "s1:now_time" => $now_time, + "s2:timing::next_sum_check" => $anvil->data->{timing}{next_sum_check}, + "s3:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check}, }}); - if ($now_time >= $next_check) + + # Time to check if the files on disk have changed? + if ($now_time >= $anvil->data->{timing}{next_sum_check}) { # Even if it is time to check, don't if a job is running. - if ((not $anvil->data->{sys}{jobs_running}) && ($anvil->Storage->check_md5sums)) + if ((not $anvil->data->{timing}{jobs_running}) && ($anvil->Storage->check_md5sums)) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0014"}); $anvil->nice_exit({code => 1}); @@ -152,33 +188,44 @@ while(1) $check_if_database_is_configured = 1; # Update the next check time. - $next_check = $now_time + $system_check; + $anvil->data->{timing}{next_sum_check} = $now_time + $anvil->data->{timing}{sum_check_interval}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - "s1:system_check" => $system_check, - "s2:next_check" => $next_check, + "s1:timing::sum_check_interval" => $anvil->data->{timing}{sum_check_interval}, + "s2:timing::next_sum_check" => $anvil->data->{timing}{next_sum_check}, }}); } - # Exit if 'run-once' selected. - if ($anvil->data->{switches}{'run-once'}) + ### NOTE: We call it once/day, but this will also trigger on restart of anvil-daemon. As such, we + ### don't use '--force' and let striker-manage-install-target skip the repo update if it happened + ### recently enough. + # Is it time to refresh RPM packages on Install Target hosts? + if ($now_time >= $anvil->data->{timing}{next_repo_check}) { - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "warn", key => "message_0055"}); - $anvil->nice_exit({code => 0}); + # Record a job, don't call it directly. It takes too long to run. + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + file => $THIS_FILE, + line => __LINE__, + job_command => $anvil->data->{path}{exe}{'striker-manage-install-target'}." --refresh", + job_data => "", + job_name => "install-target::refresh", + job_title => "job_0015", + job_description => "job_0017", + job_progress => 0, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { job_uuid => $job_uuid }}); + + # Update the next check time. + $anvil->data->{timing}{next_repo_check} = $now_time + $anvil->data->{timing}{repo_update_interval}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { + "s1:timing::repo_update_interval" => $anvil->data->{timing}{repo_update_interval}, + "s2:timing::next_repo_check" => $anvil->data->{timing}{next_repo_check}, + }}); } - # Disconnect from the database(s) and sleep now. - $anvil->Database->disconnect(); - sleep(1); + return(0); } -$anvil->nice_exit({code => 0}); - - -############################################################################################################# -# Functions # -############################################################################################################# - -# This calls anvil-manage-install-target to see if the dhcpd is running or not. If it is or isn't, the config +# This calls striker-manage-install-target to see if the dhcpd is running or not. If it is or isn't, the config # variable 'install-target::enabled' is set/updated. On non-Striker hosts, this simply returns without doing # anything. sub check_install_target @@ -193,7 +240,7 @@ sub check_install_target return(0); } - my $shell_call = $anvil->data->{path}{exe}{'anvil-manage-install-target'}." --status"; + my $shell_call = $anvil->data->{path}{exe}{'striker-manage-install-target'}." --status"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }}); my $status = "unavailable"; my $output = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__}); @@ -417,7 +464,7 @@ sub run_jobs my $jobs_file = "{\"jobs\":[\n"; # Get a list of pending or incomplete jobs. - my $return = $anvil->Database->get_jobs({debug => 3, ended_within => 300}); + my $return = $anvil->Database->get_jobs({ended_within => 300}); my $count = @{$return}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 'return' => $return, @@ -571,7 +618,7 @@ sub run_jobs my $command = $job_command." --job-uuid ".$job_uuid; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "log_0210", variables => { command => $command }}); $anvil->data->{jobs}{handles}{$job_uuid} = $anvil->System->call({ - debug => 3, + debug => 2, background => 1, stdout_file => "/tmp/anvil.job.".$job_uuid.".stdout", stderr_file => "/tmp/anvil.job.".$job_uuid.".stderr", diff --git a/tools/anvil-configure-striker b/tools/striker-configure-host similarity index 100% rename from tools/anvil-configure-striker rename to tools/striker-configure-host diff --git a/tools/anvil-manage-install-target b/tools/striker-manage-install-target similarity index 93% rename from tools/anvil-manage-install-target rename to tools/striker-manage-install-target index bb2bc4cf..476a6ba7 100755 --- a/tools/anvil-manage-install-target +++ b/tools/striker-manage-install-target @@ -53,8 +53,10 @@ $anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf' $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); # Read switches -$anvil->data->{switches}{'y'} = ""; +$anvil->data->{switches}{'y'} = ""; $anvil->data->{switches}{'job-uuid'} = ""; +$anvil->data->{switches}{refresh} = 0; +$anvil->data->{switches}{force} = ""; $anvil->Get->switches(); # Make sure we're running as 'root' @@ -95,8 +97,9 @@ if (not $anvil->data->{sys}{database}{connections}) $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"}); $anvil->nice_exit({exit_code => 8}); } +update_progress($anvil, 1, "clear"); +update_progress($anvil, 2, "log_0239"); -### NOTE: Enabling is handled at the end, after checks/updates are applied # If we're being asked to disable, just do so and exit. if ($anvil->data->{switches}{disable}) { @@ -132,17 +135,8 @@ if ($anvil->data->{switches}{disable}) ### NOTE: There is no complex job processing here. If we've been asked to enable or disable, it only ### takes a second to do it. - # If we're being called as a job, mark that we've picked it up - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job_uuid" => $anvil->data->{switches}{'job-uuid'} }}); - if ($anvil->data->{switches}{'job-uuid'}) - { - $anvil->Job->update_progress({ - debug => 1, - progress => 100, - message => $job_message, - job_uuid => $anvil->data->{switches}{'job-uuid'}, - }); - } + update_progress($anvil, 100, $job_message); + $anvil->nice_exit({exit_code => $exit_code}); } @@ -153,6 +147,8 @@ if (not $configured) { print $anvil->Words->string({key => "error_0046"})."\n"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0046"}); + + update_progress($anvil, 100, "log_0240"); $anvil->nice_exit({exit_code => 9}); } @@ -171,22 +167,19 @@ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list if ($anvil->_short_hostname !~ /striker/) { print $anvil->Words->string({key => "error_0044"})."\n"; + update_progress($anvil, 100, "error_0044"); $anvil->nice_exit({code => 4}); } -# If this isn't being called as a job, check/update the configuration and packages. -$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job_uuid" => $anvil->data->{switches}{'job-uuid'} }}); -if (not $anvil->data->{switches}{'job-uuid'}) -{ - print $anvil->Words->string({key => "message_0102"})."\n"; - load_packages($anvil); +# Calling with 'refresh' takes time, so we only do it when asked. +print $anvil->Words->string({key => "message_0102"})."\n"; +load_packages($anvil); - # Setup PXE/tftp/dhcpd config. - setup_boot_environment($anvil); +# Setup PXE/tftp/dhcpd config. +setup_boot_environment($anvil); - # Store the RPMs in repo directory. - update_install_source($anvil); -} +# Store the RPMs in repo directory. +update_install_source($anvil); # If we've been asked to enable or disable the install target, do so now. if ($anvil->data->{switches}{enable}) @@ -223,25 +216,18 @@ if ($anvil->data->{switches}{enable}) ### NOTE: There is no complex job processing here. If we've been asked to enable or disable, it only ### takes a second to do it. - # If we're being called as a job, mark that we've picked it up - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job_uuid" => $anvil->data->{switches}{'job-uuid'} }}); - if ($anvil->data->{switches}{'job-uuid'}) - { - $anvil->Job->update_progress({ - debug => 1, - progress => 100, - message => $job_message, - job_uuid => $anvil->data->{switches}{'job-uuid'}, - }); - } + update_progress($anvil, 90, $job_message); + if ($exit_code) { + update_progress($anvil, 100, ""); $anvil->nice_exit({exit_code => $exit_code}); } } # We're done print $anvil->Words->string({key => "message_0025"})."\n"; +update_progress($anvil, 100, "message_0025"); $anvil->nice_exit({exit_code => 0}); @@ -249,6 +235,25 @@ $anvil->nice_exit({exit_code => 0}); # Private functions. # ############################################################################################################# +# If this is being called as a job, this will allow the progress to be updated. +sub update_progress +{ + my ($anvil, $progress, $message) = @_; + + if (not $anvil->data->{switches}{'job-uuid'}) + { + return(0); + } + + $anvil->Job->update_progress({ + progress => $progress, + message => $message, + job_uuid => $anvil->data->{switches}{'job-uuid'}, + }); + + return(0); +} + sub check_refresh { my ($anvil) = @_; @@ -258,11 +263,19 @@ sub check_refresh $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "path::directories::packages" => $anvil->data->{path}{directories}{packages} }}); # Default to 'no' - $anvil->data->{switches}{refresh} = 0 if not defined $anvil->data->{switches}{refresh}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::refresh" => $anvil->data->{switches}{refresh} }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "switches::refresh" => $anvil->data->{switches}{refresh}, + "switches::force" => $anvil->data->{switches}{force}, + }}); + + # If refresh isn't set, we're out. + if (not $anvil->data->{switches}{refresh}) + { + return(0); + } - # It it's set, the user asked for it. - if ($anvil->data->{switches}{refresh}) + # It it's forced, the user is insisting on it. + if ($anvil->data->{switches}{force}) { return(0); } @@ -273,6 +286,9 @@ sub check_refresh if (not $anvil->data->{'install-manifest'}{'refresh-packages'}) { # We're out. + $anvil->data->{switches}{refresh} = 0; + print $anvil->Words->string({key => "log_0235"})."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0235"}); return(0); } @@ -281,19 +297,12 @@ sub check_refresh { # Source isn't configured, set it up $anvil->data->{switches}{refresh} = 1; + print $anvil->Words->string({key => "log_0237"})."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0237"}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::refresh" => $anvil->data->{switches}{refresh} }}); return(0); } - # If the user has asked to enable or disable, skip. - $anvil->data->{switches}{enable} = "" if not defined $anvil->data->{switches}{enable}; - $anvil->data->{switches}{disable} = "" if not defined $anvil->data->{switches}{disable}; - if (($anvil->data->{switches}{enable}) or ($anvil->data->{switches}{disable})) - { - # Skip. - return(0); - } - # See when the last time we refreshed the local package cache my ($unixtime, $variable_uuid, $modified_date) = $anvil->Database->read_variable({ variable_name => "install-target::refreshed", @@ -323,7 +332,7 @@ sub check_refresh }}); if ($anvil->data->{'install-manifest'}{'refresh-period'} =~ /\D/) { - $anvil->data->{'install-manifest'}{'refresh-period'} = 86400; + $anvil->data->{'install-manifest'}{'refresh-period'} = 86400; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'install-manifest::refresh-period' => $anvil->data->{'install-manifest'}{'refresh-period'}, }}); @@ -338,13 +347,17 @@ sub check_refresh }}); if ((not $variable_uuid) or ($unixtime !~ /^\d+/) or ($difference < 0)) { + # It's been long enough (or it's the first time), refresh. $anvil->data->{switches}{refresh} = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::refresh" => $anvil->data->{switches}{refresh} }}); + my $variables = { seconds => $anvil->Convert->add_commas({number => $anvil->data->{'install-manifest'}{'refresh-period'}}) }; + print $anvil->Words->string({key => "log_0239", variables => $variables})."\n"; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0239", variables => $variables}); return(0); } elsif ($difference > 0) { # Log when the next scan will happen + $anvil->data->{switches}{refresh} = 0; my $variables = { next_refresh => $anvil->Convert->add_commas({number => $difference}) }; print $anvil->Words->string({key => "log_0236", variables => $variables})."\n"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0236", variables => $variables}); @@ -398,6 +411,7 @@ sub setup_boot_environment { # Can't set this up yet. print $anvil->Words->string({key => "error_0042"})."\n"; + update_progress($anvil, 100, "error_0042"); $anvil->nice_exit({code => 2}); } @@ -459,6 +473,7 @@ sub setup_boot_environment { # Failed. print $anvil->Words->string({key => "error_0043", variables => { file => $anvil->data->{path}{configs}{'dhcpd.conf'} }})."\n"; + update_progress($anvil, 100, "error_0043"); $anvil->nice_exit({code => 3}); } elsif ($dhcpd_conf_success eq "1") @@ -468,14 +483,15 @@ sub setup_boot_environment daemon => "dhcpd", file => $anvil->data->{path}{configs}{'dhcpd.conf'}, }})."\n"; + update_progress($anvil, 10, "message_0095,!!daemon!dhcpd!!,!!file!".$anvil->data->{path}{configs}{'dhcpd.conf'}."!!"); $anvil->System->restart_daemon({daemon => $anvil->data->{sys}{daemon}{dhcpd}}); } elsif ($dhcpd_conf_success eq "2") { # Update not needed. print $anvil->Words->string({key => "message_0096", variables => { file => $anvil->data->{path}{configs}{'dhcpd.conf'} }})."\n"; + update_progress($anvil, 10, "message_0096,!!file!".$anvil->data->{path}{configs}{'dhcpd.conf'}."!!"); } - ### PXE BIOS 'default' file. my $bios_default_body = $anvil->Template->get({file => "pxe.txt", show_name => 0, name => "tftp_bios", variables => { base_url => $base_url }}); @@ -492,6 +508,7 @@ sub setup_boot_environment { # Failed. print $anvil->Words->string({key => "error_0043", variables => { file => $anvil->data->{path}{configs}{pxe_default} }})."\n"; + update_progress($anvil, 100, "error_0043,!!file!".$anvil->data->{path}{configs}{pxe_default}."!!"); $anvil->nice_exit({code => 3}); } elsif ($dhcpd_conf_success eq "1") @@ -506,9 +523,11 @@ sub setup_boot_environment } ### Generate kickstart files. + my $progress = 10; foreach my $type ("striker", "node", "dr") { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type }}); + $progress += 2; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { type => $type, progress => $progress }}); my $say_type = "#!string!message_0115!#"; if ($type eq "node") { @@ -522,7 +541,6 @@ sub setup_boot_environment type => $type, say_type => $say_type, hostname => "new-".$type.".".$domain, - date => $anvil->Get->date_and_time({date_only => 1}), os => $anvil->data->{host_os}{os_type}, url => $base_url."/os/", keyboard => $anvil->data->{kickstart}{keyboard} ? $anvil->data->{kickstart}{keyboard} : $anvil->data->{defaults}{kickstart}{keyboard}, @@ -544,6 +562,7 @@ sub setup_boot_environment { # Failed. print $anvil->Words->string({key => "error_0043", variables => { file => $kickstart_file }})."\n"; + update_progress($anvil, 100, "error_0043,!!file!".$kickstart_file."!!"); $anvil->nice_exit({code => 3}); } elsif ($kickstart_success eq "1") @@ -553,18 +572,22 @@ sub setup_boot_environment $anvil->Storage->change_owner({target => $kickstart_file, user => "apache", group => "apache" }); print $anvil->Words->string({key => "message_0097", variables => { file => $kickstart_file }})."\n"; + update_progress($anvil, $progress, "message_0097,!!file!".$kickstart_file."!!"); } elsif ($kickstart_success eq "2") { # Update not needed. print $anvil->Words->string({key => "message_0096", variables => { file => $kickstart_file }})."\n"; + update_progress($anvil, $progress, "message_0096,!!file!".$kickstart_file."!!"); } + # progress is '16' as it leaves this loop } # Configure apache to show hidden (dot) files. my $old_autoindex_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'autoindex.conf'}}); my $new_autoindex_conf = ""; my $update_autoindex_conf = 0; + $progress = 18; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_autoindex_conf => $old_autoindex_conf }}); foreach my $line (split/\n/, $old_autoindex_conf) { @@ -630,16 +653,19 @@ sub setup_boot_environment # Something went wrong. print $anvil->Words->string({key => "log_0233", variables => { file => $anvil->data->{path}{configs}{'autoindex.conf'}, 'return' => $return }})."\n"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0233", variables => { file => $anvil->data->{path}{configs}{'autoindex.conf'}, 'return' => $return }}); + update_progress($anvil, "100", "log_0233,!!file!".$anvil->data->{path}{configs}{'autoindex.conf'}."!!,!!return!".$return."!!"); $anvil->nice_exit({code => 3}); } print $anvil->Words->string({key => "message_0095", variables => { daemon => "httpd", file => $anvil->data->{path}{configs}{'autoindex.conf'} }})."\n"; $anvil->System->reload_daemon({daemon => $anvil->data->{sys}{daemon}{httpd}}); + update_progress($anvil, $progress, "message_0095,!!daemon!httpd!!,!!file!".$anvil->data->{path}{configs}{'autoindex.conf'}."!!"); } else { # Update not needed. print $anvil->Words->string({key => "message_0096", variables => { file => $anvil->data->{path}{configs}{'autoindex.conf'} }})."\n"; + update_progress($anvil, $progress, "message_0096,!!file!".$anvil->data->{path}{configs}{'autoindex.conf'}."!!"); } # Update dnf to save downloaded files for later repo building efficiency and to skip upstream repos @@ -648,6 +674,7 @@ sub setup_boot_environment my $skip_seen = 0; my $old_dnf_conf = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'dnf.conf'}}); my $new_dnf_conf = ""; + $progress = 20; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { old_dnf_conf => $old_dnf_conf }}); foreach my $line (split/\n/, $old_dnf_conf) { @@ -694,45 +721,56 @@ sub setup_boot_environment # Something went wrong. print $anvil->Words->string({key => "log_0233", variables => { file => $anvil->data->{path}{configs}{'dnf.conf'}, 'return' => $return }})."\n"; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0233", variables => { file => $anvil->data->{path}{configs}{'dnf.conf'}, 'return' => $return }}); + update_progress($anvil, 100, "log_0023,!!file!".$anvil->data->{path}{configs}{'dnf.conf'}."!!,!!refresh!return!".$return."!!"); $anvil->nice_exit({code => 3}); } print $anvil->Words->string({key => "message_0097", variables => { file => $anvil->data->{path}{configs}{'dnf.conf'} }})."\n"; + update_progress($anvil, $progress, "message_0097,!!file!".$anvil->data->{path}{configs}{'dnf.conf'}."!!"); } else { # Update not needed. print $anvil->Words->string({key => "message_0096", variables => { file => $anvil->data->{path}{configs}{'dnf.conf'} }})."\n"; + update_progress($anvil, $progress, "message_0096,!!file!".$anvil->data->{path}{configs}{'dnf.conf'}."!!"); } ### Check that daemons are enabled/disabled. ### NOTE: We don't manage dhcpd. We leave it off, but if the user enabled it, respect that. # Make sure tftp is enabled. + $progress = 22; if (-e $anvil->data->{path}{systemd}{tftp_enabled_symlink}) { print $anvil->Words->string({key => "message_0099", variables => { daemon => $anvil->data->{sys}{daemon}{tftp} }})."\n"; + update_progress($anvil, $progress, "message_0099,!!daemon!".$anvil->data->{sys}{daemon}{tftp}."!!"); } else { print $anvil->Words->string({key => "message_0098", variables => { daemon => $anvil->data->{sys}{daemon}{tftp} }})."\n"; + update_progress($anvil, $progress, "message_0098,!!daemon!".$anvil->data->{sys}{daemon}{tftp}."!!"); $anvil->System->enable_daemon({daemon => $anvil->data->{sys}{daemon}{tftp}}); $anvil->System->start_daemon({daemon => $anvil->data->{sys}{daemon}{tftp}}); } + $progress = 24; if (-e $anvil->data->{path}{systemd}{httpd_enabled_symlink}) { print $anvil->Words->string({key => "message_0099", variables => { daemon => $anvil->data->{sys}{daemon}{httpd} }})."\n"; + update_progress($anvil, $progress, "message_0099,!!daemon!".$anvil->data->{sys}{daemon}{httpd}."!!"); } else { print $anvil->Words->string({key => "message_0098", variables => { daemon => $anvil->data->{sys}{daemon}{httpd} }})."\n"; + update_progress($anvil, $progress, "message_0098,!!daemon!".$anvil->data->{sys}{daemon}{httpd}."!!"); $anvil->System->enable_daemon({daemon => $anvil->data->{sys}{daemon}{httpd}}); $anvil->System->start_daemon({daemon => $anvil->data->{sys}{daemon}{httpd}}); } # Make sure the syslinux files needed for creating the PXE boot menu are in place. + $progress = 26; if (not -e $anvil->data->{path}{directories}{tftpboot}."/vesamenu.c32") { # Copy the syslinux files print $anvil->Words->string({key => "message_0100"})."\n"; + update_progress($anvil, $progress, "message_0100"); $anvil->Storage->rsync({ debug => 2, source => $anvil->data->{path}{directories}{syslinux}."/*", @@ -742,6 +780,7 @@ sub setup_boot_environment else { print $anvil->Words->string({key => "message_0101"})."\n"; + update_progress($anvil, $progress, "message_0101"); } ### TODO: Add UEFI @@ -801,6 +840,7 @@ sub check_alteeve_repo { # Die print $anvil->Words->string({key => "error_0041"})."\n"; + update_progress($anvil, 100, "error_0041"); $anvil->nice_exit({code => 1}); } @@ -811,40 +851,50 @@ sub check_alteeve_repo sub update_install_source { my ($anvil) = @_; + + # Job progress is at '26' coming into here # Should we refresh the local repo? check_refresh($anvil); if (not $anvil->data->{switches}{refresh}) { return(0); + update_progress($anvil, 90, ""); } + ### TODO: Make sure this handles no internet access gracefully. # Clear the dnf cache - my $success = 1; - my $output = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{dnf}." clean expire-cache" }); + my $success = 1; + my $progress = 30; + my $output = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{dnf}." clean expire-cache" }); + my $packages = "/var/www/html/".$anvil->data->{host_os}{os_type}."/".$anvil->data->{host_os}{os_arch}."/os/Packages/*"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output }}); - print $anvil->Words->string({key => "message_0077", variables => { - directory => "/var/www/html/".$anvil->data->{host_os}{os_type}."/".$anvil->data->{host_os}{os_arch}."/os/Packages/*", - }})."\n"; + print $anvil->Words->string({key => "message_0077", variables => { directory => $packages }})."\n"; + update_progress($anvil, $progress, "message_0077,!!directory!".$packages."!!"); # Loop through each letter directory foreach my $letter (sort {$a cmp $b} keys %{$anvil->data->{packages}}) { + $progress += 2; + $progress = 90 if $progress > 90; my $download_path = "/var/www/html/".$anvil->data->{host_os}{os_type}."/".$anvil->data->{host_os}{os_arch}."/os/Packages/".$letter; my $array_size = @{$anvil->data->{packages}{$letter}}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { letter => $letter, download_path => $download_path, array_size => $array_size, + progress => $progress, }}); if (not -e $download_path) { $anvil->Storage->make_directory({debug => 2, directory => $download_path, mode => "0775"}); } + my $say_packages = $anvil->Convert->add_commas({number => $array_size}); print $anvil->Words->string({key => "message_0120", variables => { directory => $download_path, - packages => $anvil->Convert->add_commas({number => $array_size}), + packages => $say_packages, }})."\n"; + update_progress($anvil, $progress, "message_0120,!!directory!".$download_path."!!,!!packages!".$say_packages."!!"); my $shell_call = $anvil->data->{path}{exe}{dnf}." download --destdir ".$download_path." "; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); @@ -927,36 +977,45 @@ sub update_install_source # Something went wrong, exit. print $anvil->Words->string({key => "error_0045", variables => { error => $error_out }})."\n"; + update_progress($anvil, 100, "error_0045,!!error!".$error_out."!!"); $anvil->nice_exit({code => 7}); } + # Progress is '82' leaving this loop } # Create the repodata print $anvil->Words->string({key => "message_0118"})."\n"; my $repo_path = "/var/www/html/".$anvil->data->{host_os}{os_type}."/".$anvil->data->{host_os}{os_arch}."/os"; my $comps_xml = $repo_path."/comps.xml"; + my $target = $repo_path."/repodata/comps.xml"; if (not -e $comps_xml) { # We can't install properly without the comps.xml file, it provides grouping needed by the # guest OS install. print $anvil->Words->string({key => "message_0119", variables => { comps_xml => $comps_xml }})."\n"; + update_progress($anvil, 100, "message_0119,!!comps_xml!".$comps_xml."!!"); $anvil->nice_exit({code => 6}); } $anvil->Storage->copy_file({ debug => 2, source_file => $comps_xml, - target_file => $repo_path."/repodata/comps.xml", + target_file => $target, overwrite => 1, }); - if (not -e $repo_path."/repodata/comps.xml") + if (not -e $target) { # Something appears to have happened and it failed to copy. - print $anvil->Words->string({key => "message_0119", variables => { comps_xml => $comps_xml }})."\n"; + print $anvil->Words->string({key => "message_0129", variables => { comps_xml => $comps_xml, target => $target }})."\n"; + update_progress($anvil, 100, "message_0129,!!comps_xml!".$comps_xml."!!,!!target!".$target."!!"); $anvil->nice_exit({code => 6}); } + update_progress($anvil, 85, ""); + $output = ""; $output = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{createrepo}." -g ".$comps_xml." ".$repo_path }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output }}); + print $anvil->Words->string({key => "message_0130"})."\n"; + update_progress($anvil, 90, "message_0130"); # Update the refresh time to now. $anvil->Database->insert_or_update_variables({ @@ -969,7 +1028,6 @@ sub update_install_source variable_source_uuid => $anvil->Get->host_uuid, variable_source_table => "hosts", }); - return(0); } @@ -2659,6 +2717,7 @@ sub load_packages "zlib.x86_64", ], }; + update_progress($anvil, 5, "log_0241"); return(0); }