From 5d91211dff3030b95bcbb8c557d845aef6cf4671 Mon Sep 17 00:00:00 2001 From: Digimer Date: Thu, 4 Apr 2019 12:27:27 -0400 Subject: [PATCH] * Continued work on 'anvil-download-file', in-progress adding support for job processing. * Updated Database->insert_or_update_jobs() to use job_data when looking for a job uuid. Also fixed logging and adapted 'jobs::X' variable feeding to prevent them being undefined. Also made the search find jobs with the program name anywhere in the string, instead of just the start of the strong. * Started work on Jobs->update_progress() to handle updating downloading file information. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + Anvil/Tools/Database.pm | 4 +- Anvil/Tools/Job.pm | 63 ++++++++++++----- share/words.xml | 5 ++ tools/anvil-download-file | 141 +++++++++++++++++++++++++++++++++++--- 5 files changed, 184 insertions(+), 30 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index c118f2ff..9b715ffc 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -972,6 +972,7 @@ sub _set_paths exe => { 'anvil-change-password' => "/usr/sbin/anvil-change-password", 'anvil-daemon' => "/usr/sbin/anvil-daemon", + 'anvil-download-file' => "/usr/sbin/anvil-download-file", 'anvil-file-details' => "/usr/sbin/anvil-file-details", 'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode", 'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall", diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 65097376..51576048 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -3494,7 +3494,7 @@ sub insert_or_update_jobs $problem = 1; } - # Job name? + # Job title? if (not $job_title) { $anvil->Log->entry({source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_jobs()", parameter => "job_title" }}); @@ -3527,6 +3527,8 @@ WHERE job_name = ".$anvil->Database->quote($job_name)." AND job_command = ".$anvil->Database->quote($job_command)." +AND + job_data = ".$anvil->Database->quote($job_data)." AND job_host_uuid = ".$anvil->Database->quote($job_host_uuid)." ;"; diff --git a/Anvil/Tools/Job.pm b/Anvil/Tools/Job.pm index 09b112fa..4c96da28 100644 --- a/Anvil/Tools/Job.pm +++ b/Anvil/Tools/Job.pm @@ -166,6 +166,13 @@ sub get_job_details job_uuid => $job_uuid, }}); + # + if ((not $job_uuid) && ($anvil->data->{switches}{'job-uuid'})) + { + $job_uuid = $anvil->data->{switches}{'job-uuid'}; + + } + if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}})) { # It's not a UUID. @@ -192,11 +199,11 @@ FROM WHERE job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'})." ;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { results => $results, count => $count, }}); @@ -208,18 +215,18 @@ WHERE # If we're here, we're good. Load the details $anvil->data->{jobs}{job_uuid} = $job_uuid; - $anvil->data->{jobs}{job_host_uuid} = $results->[0]->[0]; - $anvil->data->{jobs}{job_command} = $results->[0]->[1]; - $anvil->data->{jobs}{job_data} = $results->[0]->[1]; - $anvil->data->{jobs}{job_picked_up_by} = $results->[0]->[2]; - $anvil->data->{jobs}{job_picked_up_at} = $results->[0]->[3]; - $anvil->data->{jobs}{job_updated} = $results->[0]->[4]; - $anvil->data->{jobs}{job_name} = $results->[0]->[5]; - $anvil->data->{jobs}{job_progress} = $results->[0]->[6]; - $anvil->data->{jobs}{job_title} = $results->[0]->[7]; - $anvil->data->{jobs}{job_description} = $results->[0]->[8]; - $anvil->data->{jobs}{job_status} = $results->[0]->[9]; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + $anvil->data->{jobs}{job_host_uuid} = defined $results->[0]->[0] ? $results->[0]->[0] : ""; + $anvil->data->{jobs}{job_command} = defined $results->[0]->[1] ? $results->[0]->[1] : ""; + $anvil->data->{jobs}{job_data} = defined $results->[0]->[2] ? $results->[0]->[2] : ""; + $anvil->data->{jobs}{job_picked_up_by} = defined $results->[0]->[3] ? $results->[0]->[3] : ""; + $anvil->data->{jobs}{job_picked_up_at} = defined $results->[0]->[4] ? $results->[0]->[4] : ""; + $anvil->data->{jobs}{job_updated} = defined $results->[0]->[5] ? $results->[0]->[5] : ""; + $anvil->data->{jobs}{job_name} = defined $results->[0]->[6] ? $results->[0]->[6] : ""; + $anvil->data->{jobs}{job_progress} = defined $results->[0]->[7] ? $results->[0]->[7] : ""; + $anvil->data->{jobs}{job_title} = defined $results->[0]->[8] ? $results->[0]->[8] : ""; + $anvil->data->{jobs}{job_description} = defined $results->[0]->[9] ? $results->[0]->[9] : ""; + $anvil->data->{jobs}{job_status} = defined $results->[0]->[10] ? $results->[0]->[10] : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "jobs::job_uuid" => $anvil->data->{jobs}{job_uuid}, "jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid}, "jobs::job_command" => $anvil->data->{jobs}{job_command}, @@ -234,8 +241,6 @@ WHERE "jobs::job_status" => $anvil->data->{jobs}{job_status}, }}); - # Have we been asked to check another job is already running? - # See if the job was picked up by another running instance. my $job_picked_up_by = $anvil->data->{jobs}{job_picked_up_by}; if (($check) && ($job_picked_up_by)) @@ -297,7 +302,7 @@ SELECT FROM jobs WHERE - job_command LIKE ".$anvil->Database->quote($program."%")." + job_command LIKE ".$anvil->Database->quote("%".$program."%")." AND job_progress != '100' AND @@ -453,7 +458,6 @@ sub html_list } } - return($jobs_list); } @@ -636,6 +640,29 @@ WHERE $job_status =~ s/message_0058,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "<< job_status" => $job_status }}); } + # This is used by 'anvil-download-file' + if ($job_status =~ /message_0142/gs) + { + ### NOTE: Left off here. +# my $downloaded = $anvil->data->{counts}{downloaded} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{downloaded}}) : 0; +# my $installed = $anvil->data->{counts}{installed} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{installed}}) : 0; +# my $verified = $anvil->data->{counts}{verified} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{verified}}) : 0; +# my $lines = $anvil->data->{counts}{lines} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{lines}}) : 0; +# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { +# "s1:counts::downloaded" => $anvil->data->{counts}{downloaded}, +# "s2:downloaded" => $downloaded, +# "s3:counts::installed" => $anvil->data->{counts}{installed}, +# "s4:installed" => $installed, +# "s5:counts::verified" => $anvil->data->{counts}{verified}, +# "s6:verified" => $verified, +# "s7:counts::lines" => $anvil->data->{counts}{lines}, +# "s8:lines" => $lines, +# }}); +# +# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { ">> job_status" => $job_status }}); +# $job_status =~ s/message_0142,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm; +# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "<< job_status" => $job_status }}); + } $job_uuid = $anvil->Database->insert_or_update_jobs({ file => $THIS_FILE, diff --git a/share/words.xml b/share/words.xml index 9b6aa98d..2c250391 100644 --- a/share/words.xml +++ b/share/words.xml @@ -246,6 +246,7 @@ About to try to download aproximately: [#!variable!packages!#] packages needed t Reloading the firewall... Restarting the firewall... Changing the default zone to: [#!variable!zone!#]. + * Download progress: [#!variable!percentage!# %], Downloaded: [#!variable!downloaded!#], Current rate: [#!variable!current_rate!#], Average Rate: [#!variable!average_rate!#], Time Running: [#!variable!running_time!#], Estimated left: [#!variable!estimated_left!#]. Starting: [#!variable!program!#]. @@ -794,6 +795,8 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st Manager Install Target. Enable or disable the 'Install Target' feature. Update the 'Install Target' source files and refresh RPM repository. + Download a file + The referenced file will be downloaded by the target host. The IP address will change. You will need to reconnect after applying these changes. @@ -882,6 +885,8 @@ Failed to generate an RSA public key for the user: [#!variable!user!#]. The outp Failed to backup: [#!variable!file!#], skipping. The file to be downloaded: [#!variable!file!#], already exists. Either remove it, or call again with '--overwrite'. Something went wrong moving the downloaded file from the temporary location: [#!variable!source_file!#] to the output: [#!variable!target_file!#]. Useful errors may be above this message. + The download job with UUID: [#!variable!job_uuid!#] is not valid. + The download job with UUID: [#!variable!job_uuid!#] is already being handled by another process. Yes diff --git a/tools/anvil-download-file b/tools/anvil-download-file index fd958ac6..187cb61b 100755 --- a/tools/anvil-download-file +++ b/tools/anvil-download-file @@ -4,16 +4,19 @@ # progress of any other instances currently downloading files. # # Return codes: -# 0 = Normal exit. -# 1 = URL not found. -# 2 = The requested URL was not found on the remote server. -# 3 = The requested URL does not resolve to a known domain. -# 4 = The requested URL failed because the remote host refused the connection. -# 5 = The requested URL failed because there is no route to that host. -# 6 = Abort requested, but UUID or PID not passed -# 7 = The requested URL failed because the network is unreachable. -# 8 = The file to download already exists. -# 9 = Something went wrong moving the file from temp to the output directory. +# 0 = Normal exit. +# 1 = No database connections available +# 2 = The requested URL was not found on the remote server. +# 3 = The requested URL does not resolve to a known domain. +# 4 = The requested URL failed because the remote host refused the connection. +# 5 = The requested URL failed because there is no route to that host. +# 6 = Abort requested, but UUID or PID not passed +# 7 = The requested URL failed because the network is unreachable. +# 8 = The file to download already exists. +# 9 = Something went wrong moving the file from temp to the output directory. +# 10 = URL not found. +# 11 = The --job-uuid is invalid +# 12 = The --job-uuid is already being handled by another process # # # TODO: @@ -42,6 +45,14 @@ my $anvil = Anvil::Tools->new(); $anvil->Log->level({set => 2}); $anvil->Log->secure({set => 1}); +$anvil->Database->connect; +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"}); + $anvil->nice_exit({exit_code => 1}); +} $anvil->data->{switches}{abort} = ""; $anvil->data->{switches}{'job-uuid'} = ""; @@ -59,7 +70,14 @@ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list 'switches::url' => $anvil->data->{switches}{url}, }}); +# If I don't have --abort or --url, see if there is a job waiting +if ((not $anvil->data->{switches}{abort}) && (not $anvil->data->{switches}{url})) +{ + get_job_details($anvil); +} +die; +# Do what now? if ($anvil->data->{switches}{abort}) { # Kill the other download @@ -85,6 +103,107 @@ $anvil->nice_exit({exit_code => 0}); # Private functions. # ############################################################################################################# +# This loads a job's details, or looks for unclaimed jobs that aren't finished. +sub get_job_details +{ + my ($anvil) = @_; + + # If I don't have a job-uuid, see if any jobs are pending + if (not $anvil->data->{switches}{'job-uuid'}) + { + my $job_uuid = $anvil->Job->get_job_uuid({debug => 2, program => $THIS_FILE}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + if ($anvil->Validate->is_uuid({uuid => $job_uuid})) + { + # Got one! + $anvil->data->{switches}{'job-uuid'} = $job_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'} }}); + } + } + + # If we've got a job-uuid, load the details. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'} }}); + if ($anvil->data->{switches}{'job-uuid'}) + { + my $problem = $anvil->Job->get_job_details({ + debug => 2, + check => 1, + }); + # If 'problem' is '1', the job-uuid is bad. If it's '2', it's already being handled by + # another process. In either case, we're not going to handle this. + if (not $problem) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + problem => $problem, + "jobs::job_uuid" => $anvil->data->{jobs}{job_uuid}, + "jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid}, + "jobs::job_command" => $anvil->data->{jobs}{job_command}, + "jobs::job_data" => $anvil->data->{jobs}{job_data}, + "jobs::job_picked_up_by" => $anvil->data->{jobs}{job_picked_up_by}, + "jobs::job_picked_up_at" => $anvil->data->{jobs}{job_picked_up_at}, + "jobs::job_updated" => $anvil->data->{jobs}{job_updated}, + "jobs::job_name" => $anvil->data->{jobs}{job_name}, + "jobs::job_progress" => $anvil->data->{jobs}{job_progress}, + "jobs::job_title" => $anvil->data->{jobs}{job_title}, + "jobs::job_description" => $anvil->data->{jobs}{job_description}, + "jobs::job_status" => $anvil->data->{jobs}{job_status}, + }}); + + # Store the job-uuid in the switches hash. + $anvil->data->{switches}{'job-uuid'} = $anvil->data->{jobs}{job_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }}); + } + elsif ($problem eq "1") + { + # Bad UUID + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0061", variables => { job_uuid => $anvil->data->{jobs}{job_uuid} }}); + $anvil->nice_exit({exit_code => 11}); + } + else + { + # Job is already being handled by another active process. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0062", variables => { job_uuid => $anvil->data->{jobs}{job_uuid} }}); + $anvil->nice_exit({exit_code => 12}); + } + + # Pull apart the job-data and feed them into the switches hash + if ($anvil->data->{switches}{'job-uuid'}) + { + foreach my $pair (split/,/, $anvil->data->{jobs}{job_data}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pair => $pair }}); + + my ($variable, $value) = ($pair =~ /^(.*?)=(.*)$/); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + variable => $variable, + value => $value, + }}); + + $anvil->data->{switches}{$variable} = $value; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::${variable}" => $anvil->data->{switches}{$variable} }}); + } + } + + # If "job_picked_up_by" is set, clear it. (It won't be running because we did 'check' + # earlier) + if ($anvil->data->{jobs}{job_picked_up_by}) + { + $anvil->Job->clear({ + debug => 2, + job_uuid => $anvil->data->{switches}{'job-uuid'}, + }); + } + + # Mark it as 1% done. + $anvil->Job->update_progress({ + debug => 2, + job_uuid => $anvil->data->{switches}{'job-uuid'}, + }); + } + + return(0); +} + sub download_file { my ($anvil) = @_; @@ -109,7 +228,7 @@ sub download_file if (($url !~ /^ftp\:\/\//) && ($url !~ /^http\:\/\//) && ($url !~ /^https\:\/\//)) { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0286", variables => { url => $url }}); - $anvil->nice_exit({exit_code => 1}); + $anvil->nice_exit({exit_code => 10}); } # If the target file exists, exit.