seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Continued work on 'anvil-download-file', in-progress adding support for job processing.

Browse Source 
* Updated Database->insert_or_update_jobs() to use job_data when looking for a job uuid. Also fixed logging and adapted 'jobs::X' variable feeding to prevent them being undefined. Also made the search find jobs with the program name anywhere in the string, instead of just the start of the strong.
* Started work on Jobs->update_progress() to handle updating downloading file information.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent 605c3c4ffb
commit 5d91211dff
5 changed files with 184 additions and 30 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      Anvil/Tools.pm
  2. 4
      Anvil/Tools/Database.pm
  3. 63
      Anvil/Tools/Job.pm
  4. 5
      share/words.xml
  5. 141
      tools/anvil-download-file

1
Anvil/Tools.pm
Unescape View File

@ -972,6 +972,7 @@ sub _set_paths
exe => { exe => {
'anvil-change-password' => "/usr/sbin/anvil-change-password", 'anvil-change-password' => "/usr/sbin/anvil-change-password",
'anvil-daemon' => "/usr/sbin/anvil-daemon", 'anvil-daemon' => "/usr/sbin/anvil-daemon",
'anvil-download-file' => "/usr/sbin/anvil-download-file",
'anvil-file-details' => "/usr/sbin/anvil-file-details", 'anvil-file-details' => "/usr/sbin/anvil-file-details",
'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode", 'anvil-maintenance-mode' => "/usr/sbin/anvil-maintenance-mode",
'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall", 'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall",

4
Anvil/Tools/Database.pm
Unescape View File

@ -3494,7 +3494,7 @@ sub insert_or_update_jobs
$problem = 1; $problem = 1;
} }
# Job name? # Job title?
if (not $job_title) if (not $job_title)
{ {
$anvil->Log->entry({source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_jobs()", parameter => "job_title" }}); $anvil->Log->entry({source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_jobs()", parameter => "job_title" }});
@ -3527,6 +3527,8 @@ WHERE
job_name = ".$anvil->Database->quote($job_name)." job_name = ".$anvil->Database->quote($job_name)."
AND AND
job_command = ".$anvil->Database->quote($job_command)." job_command = ".$anvil->Database->quote($job_command)."
AND
job_data = ".$anvil->Database->quote($job_data)."
AND AND
job_host_uuid = ".$anvil->Database->quote($job_host_uuid)." job_host_uuid = ".$anvil->Database->quote($job_host_uuid)."
;"; ;";

63
Anvil/Tools/Job.pm
Unescape View File

@ -166,6 +166,13 @@ sub get_job_details
job_uuid => $job_uuid, job_uuid => $job_uuid,
}}); }});
#
if ((not $job_uuid) && ($anvil->data->{switches}{'job-uuid'}))
{
$job_uuid = $anvil->data->{switches}{'job-uuid'};
}
if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}})) if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}}))
{ {
# It's not a UUID. # It's not a UUID.
@ -192,11 +199,11 @@ FROM
WHERE WHERE
job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'})." job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'})."
;"; ;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results, results => $results,
count => $count, count => $count,
}}); }});
@ -208,18 +215,18 @@ WHERE
# If we're here, we're good. Load the details # If we're here, we're good. Load the details
$anvil->data->{jobs}{job_uuid} = $job_uuid; $anvil->data->{jobs}{job_uuid} = $job_uuid;
$anvil->data->{jobs}{job_host_uuid} = $results->[0]->[0]; $anvil->data->{jobs}{job_host_uuid} = defined $results->[0]->[0] ? $results->[0]->[0] : "";
$anvil->data->{jobs}{job_command} = $results->[0]->[1]; $anvil->data->{jobs}{job_command} = defined $results->[0]->[1] ? $results->[0]->[1] : "";
$anvil->data->{jobs}{job_data} = $results->[0]->[1]; $anvil->data->{jobs}{job_data} = defined $results->[0]->[2] ? $results->[0]->[2] : "";
$anvil->data->{jobs}{job_picked_up_by} = $results->[0]->[2]; $anvil->data->{jobs}{job_picked_up_by} = defined $results->[0]->[3] ? $results->[0]->[3] : "";
$anvil->data->{jobs}{job_picked_up_at} = $results->[0]->[3]; $anvil->data->{jobs}{job_picked_up_at} = defined $results->[0]->[4] ? $results->[0]->[4] : "";
$anvil->data->{jobs}{job_updated} = $results->[0]->[4]; $anvil->data->{jobs}{job_updated} = defined $results->[0]->[5] ? $results->[0]->[5] : "";
$anvil->data->{jobs}{job_name} = $results->[0]->[5]; $anvil->data->{jobs}{job_name} = defined $results->[0]->[6] ? $results->[0]->[6] : "";
$anvil->data->{jobs}{job_progress} = $results->[0]->[6]; $anvil->data->{jobs}{job_progress} = defined $results->[0]->[7] ? $results->[0]->[7] : "";
$anvil->data->{jobs}{job_title} = $results->[0]->[7]; $anvil->data->{jobs}{job_title} = defined $results->[0]->[8] ? $results->[0]->[8] : "";
$anvil->data->{jobs}{job_description} = $results->[0]->[8]; $anvil->data->{jobs}{job_description} = defined $results->[0]->[9] ? $results->[0]->[9] : "";
$anvil->data->{jobs}{job_status} = $results->[0]->[9]; $anvil->data->{jobs}{job_status} = defined $results->[0]->[10] ? $results->[0]->[10] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid}, "jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
"jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid}, "jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid},
"jobs::job_command" => $anvil->data->{jobs}{job_command}, "jobs::job_command" => $anvil->data->{jobs}{job_command},
@ -234,8 +241,6 @@ WHERE
"jobs::job_status" => $anvil->data->{jobs}{job_status}, "jobs::job_status" => $anvil->data->{jobs}{job_status},
}}); }});
# Have we been asked to check another job is already running?
# See if the job was picked up by another running instance. # See if the job was picked up by another running instance.
my $job_picked_up_by = $anvil->data->{jobs}{job_picked_up_by}; my $job_picked_up_by = $anvil->data->{jobs}{job_picked_up_by};
if (($check) && ($job_picked_up_by)) if (($check) && ($job_picked_up_by))
@ -297,7 +302,7 @@ SELECT
FROM FROM
jobs jobs
WHERE WHERE
job_command LIKE ".$anvil->Database->quote($program."%")." job_command LIKE ".$anvil->Database->quote("%".$program."%")."
AND AND
job_progress != '100' job_progress != '100'
AND AND
@ -453,7 +458,6 @@ sub html_list
} }
} }
return($jobs_list); return($jobs_list);
} }
@ -636,6 +640,29 @@ WHERE
$job_status =~ s/message_0058,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm; $job_status =~ s/message_0058,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "<< job_status" => $job_status }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "<< job_status" => $job_status }});
} }
# This is used by 'anvil-download-file'
if ($job_status =~ /message_0142/gs)
{
### NOTE: Left off here.
# my $downloaded = $anvil->data->{counts}{downloaded} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{downloaded}}) : 0;
# my $installed = $anvil->data->{counts}{installed} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{installed}}) : 0;
# my $verified = $anvil->data->{counts}{verified} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{verified}}) : 0;
# my $lines = $anvil->data->{counts}{lines} ? $anvil->Convert->add_commas({number => $anvil->data->{counts}{lines}}) : 0;
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
# "s1:counts::downloaded" => $anvil->data->{counts}{downloaded},
# "s2:downloaded" => $downloaded,
# "s3:counts::installed" => $anvil->data->{counts}{installed},
# "s4:installed" => $installed,
# "s5:counts::verified" => $anvil->data->{counts}{verified},
# "s6:verified" => $verified,
# "s7:counts::lines" => $anvil->data->{counts}{lines},
# "s8:lines" => $lines,
# }});
#
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { ">> job_status" => $job_status }});
# $job_status =~ s/message_0142,!!downloaded!.*?!!,!!installed!.*?!!,!!verified!.*?!!,!!lines!.*?!!/message_0058,!!downloaded!$downloaded!!,!!installed!$installed!!,!!verified!$verified!!,!!lines!$lines!!/sm;
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "<< job_status" => $job_status }});
}
$job_uuid = $anvil->Database->insert_or_update_jobs({ $job_uuid = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE, file => $THIS_FILE,

5
share/words.xml
Unescape View File

@ -246,6 +246,7 @@ About to try to download aproximately: [#!variable!packages!#] packages needed t
<key name="message_0139">Reloading the firewall...</key> <key name="message_0139">Reloading the firewall...</key>
<key name="message_0140">Restarting the firewall...</key> <key name="message_0140">Restarting the firewall...</key>
<key name="message_0141">Changing the default zone to: [#!variable!zone!#].</key> <key name="message_0141">Changing the default zone to: [#!variable!zone!#].</key>
<key name="message_0142">* Download progress: [#!variable!percentage!# %], Downloaded: [#!variable!downloaded!#], Current rate: [#!variable!current_rate!#], Average Rate: [#!variable!average_rate!#], Time Running: [#!variable!running_time!#], Estimated left: [#!variable!estimated_left!#].</key>
<!-- Log entries --> <!-- Log entries -->
<key name="log_0001">Starting: [#!variable!program!#].</key> <key name="log_0001">Starting: [#!variable!program!#].</key>
@ -794,6 +795,8 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st
<key name="job_0015">Manager Install Target.</key> <key name="job_0015">Manager Install Target.</key>
<key name="job_0016">Enable or disable the 'Install Target' feature.</key> <key name="job_0016">Enable or disable the 'Install Target' feature.</key>
<key name="job_0017">Update the 'Install Target' source files and refresh RPM repository.</key> <key name="job_0017">Update the 'Install Target' source files and refresh RPM repository.</key>
<key name="job_0018">Download a file</key>
<key name="job_0019">The referenced file will be downloaded by the target host.</key>
<!-- Warnings --> <!-- Warnings -->
<key name="striker_warning_0001">The IP address will change. You will need to reconnect after applying these changes.</key> <key name="striker_warning_0001">The IP address will change. You will need to reconnect after applying these changes.</key>
@ -882,6 +885,8 @@ Failed to generate an RSA public key for the user: [#!variable!user!#]. The outp
<key name="error_0058">Failed to backup: [#!variable!file!#], skipping.</key> <key name="error_0058">Failed to backup: [#!variable!file!#], skipping.</key>
<key name="error_0059">The file to be downloaded: [#!variable!file!#], already exists. Either remove it, or call again with '--overwrite'.</key> <key name="error_0059">The file to be downloaded: [#!variable!file!#], already exists. Either remove it, or call again with '--overwrite'.</key>
<key name="error_0060">Something went wrong moving the downloaded file from the temporary location: [#!variable!source_file!#] to the output: [#!variable!target_file!#]. Useful errors may be above this message.</key> <key name="error_0060">Something went wrong moving the downloaded file from the temporary location: [#!variable!source_file!#] to the output: [#!variable!target_file!#]. Useful errors may be above this message.</key>
<key name="error_0061">The download job with UUID: [#!variable!job_uuid!#] is not valid.</key>
<key name="error_0062">The download job with UUID: [#!variable!job_uuid!#] is already being handled by another process.</key>
<!-- These are units, words and so on used when displaying information. --> <!-- These are units, words and so on used when displaying information. -->
<key name="unit_0001">Yes</key> <key name="unit_0001">Yes</key>

141
tools/anvil-download-file
Unescape View File

@ -4,16 +4,19 @@
# progress of any other instances currently downloading files. # progress of any other instances currently downloading files.
# #
# Return codes: # Return codes:
# 0 = Normal exit. # 0 = Normal exit.
# 1 = URL not found. # 1 = No database connections available
# 2 = The requested URL was not found on the remote server. # 2 = The requested URL was not found on the remote server.
# 3 = The requested URL does not resolve to a known domain. # 3 = The requested URL does not resolve to a known domain.
# 4 = The requested URL failed because the remote host refused the connection. # 4 = The requested URL failed because the remote host refused the connection.
# 5 = The requested URL failed because there is no route to that host. # 5 = The requested URL failed because there is no route to that host.
# 6 = Abort requested, but UUID or PID not passed # 6 = Abort requested, but UUID or PID not passed
# 7 = The requested URL failed because the network is unreachable. # 7 = The requested URL failed because the network is unreachable.
# 8 = The file to download already exists. # 8 = The file to download already exists.
# 9 = Something went wrong moving the file from temp to the output directory. # 9 = Something went wrong moving the file from temp to the output directory.
# 10 = URL not found.
# 11 = The --job-uuid is invalid
# 12 = The --job-uuid is already being handled by another process
# #
# #
# TODO: # TODO:
@ -42,6 +45,14 @@ my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2}); $anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1}); $anvil->Log->secure({set => 1});
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
$anvil->data->{switches}{abort} = ""; $anvil->data->{switches}{abort} = "";
$anvil->data->{switches}{'job-uuid'} = ""; $anvil->data->{switches}{'job-uuid'} = "";
@ -59,7 +70,14 @@ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list
'switches::url' => $anvil->data->{switches}{url}, 'switches::url' => $anvil->data->{switches}{url},
}}); }});
# If I don't have --abort or --url, see if there is a job waiting
if ((not $anvil->data->{switches}{abort}) && (not $anvil->data->{switches}{url}))
{
get_job_details($anvil);
}
die;
# Do what now?
if ($anvil->data->{switches}{abort}) if ($anvil->data->{switches}{abort})
{ {
# Kill the other download # Kill the other download
@ -85,6 +103,107 @@ $anvil->nice_exit({exit_code => 0});
# Private functions. # # Private functions. #
############################################################################################################# #############################################################################################################
# This loads a job's details, or looks for unclaimed jobs that aren't finished.
sub get_job_details
{
my ($anvil) = @_;
# If I don't have a job-uuid, see if any jobs are pending
if (not $anvil->data->{switches}{'job-uuid'})
{
my $job_uuid = $anvil->Job->get_job_uuid({debug => 2, program => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
if ($anvil->Validate->is_uuid({uuid => $job_uuid}))
{
# Got one!
$anvil->data->{switches}{'job-uuid'} = $job_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
}
}
# If we've got a job-uuid, load the details.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
if ($anvil->data->{switches}{'job-uuid'})
{
my $problem = $anvil->Job->get_job_details({
debug => 2,
check => 1,
});
# If 'problem' is '1', the job-uuid is bad. If it's '2', it's already being handled by
# another process. In either case, we're not going to handle this.
if (not $problem)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
problem => $problem,
"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
"jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid},
"jobs::job_command" => $anvil->data->{jobs}{job_command},
"jobs::job_data" => $anvil->data->{jobs}{job_data},
"jobs::job_picked_up_by" => $anvil->data->{jobs}{job_picked_up_by},
"jobs::job_picked_up_at" => $anvil->data->{jobs}{job_picked_up_at},
"jobs::job_updated" => $anvil->data->{jobs}{job_updated},
"jobs::job_name" => $anvil->data->{jobs}{job_name},
"jobs::job_progress" => $anvil->data->{jobs}{job_progress},
"jobs::job_title" => $anvil->data->{jobs}{job_title},
"jobs::job_description" => $anvil->data->{jobs}{job_description},
"jobs::job_status" => $anvil->data->{jobs}{job_status},
}});
# Store the job-uuid in the switches hash.
$anvil->data->{switches}{'job-uuid'} = $anvil->data->{jobs}{job_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
}
elsif ($problem eq "1")
{
# Bad UUID
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0061", variables => { job_uuid => $anvil->data->{jobs}{job_uuid} }});
$anvil->nice_exit({exit_code => 11});
}
else
{
# Job is already being handled by another active process.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "error_0062", variables => { job_uuid => $anvil->data->{jobs}{job_uuid} }});
$anvil->nice_exit({exit_code => 12});
}
# Pull apart the job-data and feed them into the switches hash
if ($anvil->data->{switches}{'job-uuid'})
{
foreach my $pair (split/,/, $anvil->data->{jobs}{job_data})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { pair => $pair }});
my ($variable, $value) = ($pair =~ /^(.*?)=(.*)$/);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
variable => $variable,
value => $value,
}});
$anvil->data->{switches}{$variable} = $value;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::${variable}" => $anvil->data->{switches}{$variable} }});
}
}
# If "job_picked_up_by" is set, clear it. (It won't be running because we did 'check'
# earlier)
if ($anvil->data->{jobs}{job_picked_up_by})
{
$anvil->Job->clear({
debug => 2,
job_uuid => $anvil->data->{switches}{'job-uuid'},
});
}
# Mark it as 1% done.
$anvil->Job->update_progress({
debug => 2,
job_uuid => $anvil->data->{switches}{'job-uuid'},
});
}
return(0);
}
sub download_file sub download_file
{ {
my ($anvil) = @_; my ($anvil) = @_;
@ -109,7 +228,7 @@ sub download_file
if (($url !~ /^ftp\:\/\//) && ($url !~ /^http\:\/\//) && ($url !~ /^https\:\/\//)) if (($url !~ /^ftp\:\/\//) && ($url !~ /^http\:\/\//) && ($url !~ /^https\:\/\//))
{ {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0286", variables => { url => $url }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0286", variables => { url => $url }});
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 10});
} }
# If the target file exists, exit. # If the target file exists, exit.

Write Preview
Loading…
Cancel
Save