* Created Database->insert_or_update_file_locations().

* Fixed some bugs in Database->insert_or_update_files() and the 'files' table/procedure in the SQL schema.
* Got more work done on anvil-manage-files.
* Created Job->get_job_details().
* Added an executable check for files in Storage->scan_directory().
* Cleaned up some logging and switched to Job->get_job_details() in anvil-update-states and striker-configure-host.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent d3b2f2fd35
commit 0d62a28fda
  1. 1
      Anvil/Tools.pm
  2. 421
      Anvil/Tools/Database.pm
  3. 133
      Anvil/Tools/Job.pm
  4. 5
      Anvil/Tools/Storage.pm
  5. 45
      cgi-bin/upload.pl
  6. 14
      share/anvil.sql
  7. 7
      share/words.xml
  8. 160
      tools/anvil-manage-files
  9. 3
      tools/anvil-update-states
  10. 55
      tools/anvil-update-system
  11. 75
      tools/striker-configure-host

@ -974,6 +974,7 @@ sub _set_paths
'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall",
'anvil-manage-power' => "/usr/sbin/anvil-manage-power",
'anvil-report-memory' => "/usr/sbin/anvil-report-memory",
'anvil-update-files' => "/usr/sbin/anvil-update-files",
'anvil-update-states' => "/usr/sbin/anvil-update-states",
'chmod' => "/usr/bin/chmod",
'chown' => "/usr/bin/chown",

@ -26,6 +26,8 @@ my $THIS_FILE = "Database.pm";
# initialize
# insert_or_update_bridges
# insert_or_update_bonds
# insert_or_update_file_locations
# insert_or_update_files
# insert_or_update_hosts
# insert_or_update_ip_addresses
# insert_or_update_jobs
@ -2176,6 +2178,425 @@ WHERE
return($bond_uuid);
}
=head2 insert_or_update_file_locations
This updates (or inserts) a record in the 'file_locations' table. The C<< file_location_uuid >> referencing the database row will be returned.
This table is used to track which of the files in the database are on a given host.
If there is an error, an empty string is returned.
Parameters;
=head3 file_location_uuid (optional)
The record to update, when passed.
If not passed, a check will be made to see if an existing entry is found for C<< file_name >>. If found, that entry will be updated. If not found, a new record will be inserted.
=head3 file_location_file_uuid (required)
This is the C<< files >> -> C<< file_uuid >> referenced by this record.
=head3 file_location_host_uuid (optional, default 'sys::host_uuid')
This is the C<< hosts >> -> C<< host_uuid >> referenced by this record.
=cut
sub insert_or_update_file_locations
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }});
my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : "";
my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : "";
my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : $anvil->data->{sys}{host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $file_location_host_uuid,
}});
if (not $file_location_file_uuid)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_file_uuid" }});
return("");
}
if (not $file_location_host_uuid)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_host_uuid" }});
return("");
}
# Made sure the file_uuuid and host_uuid are valid
### TODO
# If we don't have a UUID, see if we can find one for the given md5sum.
if (not $file_location_uuid)
{
my $query = "
SELECT
file_location_uuid
FROM
file_locations
WHERE
file_location_file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_file_uuid)."
AND
file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_host_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if ($count)
{
$file_location_uuid = $results->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
}
}
# If I still don't have an file_location_uuid, we're INSERT'ing .
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
if (not $file_location_uuid)
{
# It's possible that this is called before the host is recorded in the database. So to be
# safe, we'll return without doing anything if there is no host_uuid in the database.
my $hosts = $anvil->Database->get_hosts();
my $found = 0;
foreach my $hash_ref (@{$hosts})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"hash_ref->{host_uuid}" => $hash_ref->{host_uuid},
"sys::host_uuid" => $anvil->data->{sys}{host_uuid},
}});
if ($hash_ref->{host_uuid} eq $anvil->data->{sys}{host_uuid})
{
$found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { found => $found }});
}
}
if (not $found)
{
# We're out.
return("");
}
# INSERT
$file_location_uuid = $anvil->Get->uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
my $query = "
INSERT INTO
file_locations
(
file_location_uuid,
file_location_file_uuid,
file_location_host_uuid,
file_md5sum,
file_type,
modified_date
) VALUES (
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_file_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_location_host_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
);
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
}
else
{
# Query the rest of the values and see if anything changed.
my $query = "
SELECT
file_location_file_uuid,
file_location_host_uuid,
FROM
file_locations
WHERE
file_location_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if (not $count)
{
# I have a file_location_uuid but no matching record. Probably an error.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0216", variables => { uuid_name => "file_location_uuid", uuid => $file_location_uuid }});
return("");
}
foreach my $row (@{$results})
{
my $old_file_location_file_uuid = $row->[0];
my $old_file_location_host_uuid = $row->[1];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
old_file_location_file_uuid => $old_file_location_file_uuid,
old_file_location_host_uuid => $old_file_location_host_uuid,
}});
# Anything change?
if (($old_file_location_file_uuid ne $file_location_file_uuid) or
($old_file_location_host_uuid ne $file_location_host_uuid))
{
# Something changed, save.
my $query = "
UPDATE
file_locations
SET
file_location_file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_file_uuid).",
file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_host_uuid).",
modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
WHERE
file_location_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_location_uuid)."
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
}
}
}
return($file_location_uuid);
}
=head2 insert_or_update_files
This updates (or inserts) a record in the 'files' table. The C<< file_uuid >> referencing the database row will be returned.
If there is an error, an empty string is returned.
Parameters;
=head3 file_uuid (optional)
If not passed, a check will be made to see if an existing entry is found for C<< file_name >>. If found, that entry will be updated. If not found, a new record will be inserted.
=head3 file_name (required)
This is the file's name.
=head3 file_size (required)
This is the file's size in bytes. It is recorded as a quick way to determine if the file has changed on disk.
=head3 file_md5sum (requred)
This is the sum as calculated when the file is first uploaded. Once recorded, it can't change.
=head3 file_type (required)
This is the file's type/purpose. The expected values are 'iso' (disc image a new server can be installed from or mounted in a virtual optical drive), 'repo_rpm' (a package to install on a guest that provides access to Anvil! RPM software), 'script' (pre or post migration scripts), 'image' (images to use for newly created servers, instead of installing from an ISO or PXE), or 'other'.
=cut
sub insert_or_update_files
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }});
my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : "";
my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : "";
my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : "";
my $file_md5sum = defined $parameter->{file_md5sum} ? $parameter->{file_md5sum} : "";
my $file_type = defined $parameter->{file_type} ? $parameter->{file_type} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_uuid => $file_uuid,
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
}});
if (not $file_name)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_name" }});
return("");
}
if (not $file_name)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_name" }});
return("");
}
if (not $file_md5sum)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_md5sum" }});
return("");
}
if (not $file_type)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_type" }});
return("");
}
# If we don't have a UUID, see if we can find one for the given md5sum.
if (not $file_uuid)
{
my $query = "
SELECT
file_uuid
FROM
files
WHERE
file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if ($count)
{
$file_uuid = $results->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_uuid => $file_uuid }});
}
}
# If I still don't have an file_uuid, we're INSERT'ing .
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_uuid => $file_uuid }});
if (not $file_uuid)
{
# It's possible that this is called before the host is recorded in the database. So to be
# safe, we'll return without doing anything if there is no host_uuid in the database.
my $hosts = $anvil->Database->get_hosts();
my $found = 0;
foreach my $hash_ref (@{$hosts})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"hash_ref->{host_uuid}" => $hash_ref->{host_uuid},
"sys::host_uuid" => $anvil->data->{sys}{host_uuid},
}});
if ($hash_ref->{host_uuid} eq $anvil->data->{sys}{host_uuid})
{
$found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { found => $found }});
}
}
if (not $found)
{
# We're out.
return("");
}
# INSERT
$file_uuid = $anvil->Get->uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_uuid => $file_uuid }});
my $query = "
INSERT INTO
files
(
file_uuid,
file_name,
file_size,
file_md5sum,
file_type,
modified_date
) VALUES (
".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_name).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_size).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
);
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
}
else
{
# Query the rest of the values and see if anything changed.
my $query = "
SELECT
file_name,
file_size,
file_md5sum,
file_type
FROM
files
WHERE
file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if (not $count)
{
# I have a file_uuid but no matching record. Probably an error.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0216", variables => { uuid_name => "file_uuid", uuid => $file_uuid }});
return("");
}
foreach my $row (@{$results})
{
my $old_file_name = $row->[0];
my $old_file_size = $row->[1];
my $old_file_md5sum = $row->[2];
my $old_file_type = $row->[3];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
old_file_name => $old_file_name,
old_file_size => $old_file_size,
old_file_md5sum => $old_file_md5sum,
old_file_type => $old_file_type,
}});
# Anything change?
if (($old_file_name ne $file_name) or
($old_file_size ne $file_size) or
($old_file_md5sum ne $file_md5sum) or
($old_file_type ne $file_type))
{
# Something changed, save.
my $query = "
UPDATE
files
SET
file_name = ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).",
file_size = ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).",
file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
file_type = ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
WHERE
file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)."
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
}
}
}
return($file_uuid);
}
=head2 insert_or_update_hosts
This updates (or inserts) a record in the 'hosts' table. The C<< host_uuid >> UUID will be returned.

@ -13,6 +13,7 @@ my $THIS_FILE = "Job.pm";
### Methods;
# clear
# get_job_details
# get_job_uuid
# html_list
# running
@ -121,6 +122,138 @@ sub clear
return(0);
}
=head2 get_job_details
This takes a C<< job_uuid >> and returns the job's details. If the job is found, C<< 0 >> is returned. If it isn't found, C<< 1 >> is returned. If it is found, but C<< check >> was set and the process is still alice, C<< 2 >> is returned.
When successful, the job details will be stored in;
* C<< jobs::job_uuid >>
* C<< jobs::job_host_uuid >>
* C<< jobs::job_command >>
* C<< jobs::job_data >>
* C<< jobs::job_updated >>
* C<< jobs::job_picked_up_by >>
* C<< jobs::job_picked_up_at >>
* C<< jobs::job_name >>
* C<< jobs::job_progress >>
* C<< jobs::job_title >>
* C<< jobs::job_description >>
* C<< jobs::job_status >>
Parameters;
=head3 check (optional, default '1')
This checks to see if the job was picked up by a program that is still running. If set to C<< 1 >> and that process is running, this method will return C<< 2 >>. If set to C<< 0 >>, the job data will be loaded (if found) and C<< 0 >> will be returned.
=head3 job_uuid (required)
This is the job UUID to pull up.
=cut
sub get_job_details
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
my $check = defined $parameter->{check} ? $parameter->{check} : "";
my $job_uuid = defined $parameter->{job_uuid} ? $parameter->{job_uuid} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
check => $check,
job_uuid => $job_uuid,
}});
if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}}))
{
# It's not a UUID.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
return(1);
}
# If I'm here, see if we can read the job details.
my $query = "
SELECT
job_host_uuid,
job_command,
job_data,
job_picked_up_by,
job_picked_up_at,
job_updated,
job_name,
job_progress,
job_title,
job_description,
job_status
FROM
jobs
WHERE
job_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{switches}{'job-uuid'})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
if ($count < 1)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
$anvil->nice_exit({code => 2});
}
# If we're here, we're good. Load the details
$anvil->data->{jobs}{job_uuid} = $job_uuid;
$anvil->data->{jobs}{job_host_uuid} = $results->[0]->[0];
$anvil->data->{jobs}{job_command} = $results->[0]->[1];
$anvil->data->{jobs}{job_data} = $results->[0]->[1];
$anvil->data->{jobs}{job_picked_up_by} = $results->[0]->[2];
$anvil->data->{jobs}{job_picked_up_at} = $results->[0]->[3];
$anvil->data->{jobs}{job_updated} = $results->[0]->[4];
$anvil->data->{jobs}{job_name} = $results->[0]->[5];
$anvil->data->{jobs}{job_progress} = $results->[0]->[6];
$anvil->data->{jobs}{job_title} = $results->[0]->[7];
$anvil->data->{jobs}{job_description} = $results->[0]->[8];
$anvil->data->{jobs}{job_status} = $results->[0]->[9];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
"jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid},
"jobs::job_command" => $anvil->data->{jobs}{job_command},
"jobs::job_data" => $anvil->data->{jobs}{job_data},
"jobs::job_picked_up_by" => $anvil->data->{jobs}{job_picked_up_by},
"jobs::job_picked_up_at" => $anvil->data->{jobs}{job_picked_up_at},
"jobs::job_updated" => $anvil->data->{jobs}{job_updated},
"jobs::job_name" => $anvil->data->{jobs}{job_name},
"jobs::job_progress" => $anvil->data->{jobs}{job_progress},
"jobs::job_title" => $anvil->data->{jobs}{job_title},
"jobs::job_description" => $anvil->data->{jobs}{job_description},
"jobs::job_status" => $anvil->data->{jobs}{job_status},
}});
# Have we been asked to check another job is already running?
# See if the job was picked up by another running instance.
my $job_picked_up_by = $anvil->data->{jobs}{job_picked_up_by};
if (($check) && ($job_picked_up_by))
{
# Check if the PID is still active.
$anvil->System->pids({ignore_me => 1});
# Is the PID that picked up the job still alive?
if (exists $anvil->data->{pids}{$job_picked_up_by})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0146", variables => { pid => $job_picked_up_by }});
return(2);
}
}
return(0);
}
=head2 get_job_uuid
This takes the name of a program and looks in jobs for a pending job with the same command. If it is found, C<< jobs::job_uuid >> is set and the C<< job_uuid >> is returned. If no job is found, and empty string is returned.

@ -1782,6 +1782,7 @@ sub scan_directory
next if $file eq ".";
next if $file eq "..";
my $full_path = $directory."/".$file;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }});
if ((-d $full_path) && ($recursive))
{
@ -1790,7 +1791,7 @@ sub scan_directory
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"scan::directories::${full_path}::type" => $anvil->data->{scan}{directories}{$full_path}{type},
}});
$anvil->Storage->scan_directory({directory => $full_path, recursive => $recursive});
$anvil->Storage->scan_directory({debug => $debug, directory => $full_path, recursive => $recursive});
}
elsif (-l $full_path)
{
@ -1813,6 +1814,7 @@ sub scan_directory
$anvil->data->{scan}{directories}{$full_path}{size} = $details[7];
$anvil->data->{scan}{directories}{$full_path}{mtime} = $details[9];
$anvil->data->{scan}{directories}{$full_path}{mimetype} = mimetype($full_path);
$anvil->data->{scan}{directories}{$full_path}{executable} = -x $full_path ? 1 : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"scan::directories::${full_path}::type" => $anvil->data->{scan}{directories}{$full_path}{type},
"scan::directories::${full_path}::mode" => $anvil->data->{scan}{directories}{$full_path}{mode},
@ -1821,6 +1823,7 @@ sub scan_directory
"scan::directories::${full_path}::size" => $anvil->data->{scan}{directories}{$full_path}{size},
"scan::directories::${full_path}::mtime" => $anvil->data->{scan}{directories}{$full_path}{mtime},
"scan::directories::${full_path}::mimetype" => $anvil->data->{scan}{directories}{$full_path}{mimetype},
"scan::directories::${full_path}::executable" => $anvil->data->{scan}{directories}{$full_path}{executable},
}});
}
}

@ -24,6 +24,7 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Get->switches;
my $cgi = CGI->new;
print "Content-type: text/html; charset=utf-8\n\n";
@ -56,6 +57,13 @@ if ($cgi->param())
}
close $file_handle;
# TODO: Call anvil-manage-files as a backgroup process, use the logic below and move the
$anvil->System->call({
debug => 2,
background => 1,
shell_call => $anvil->data->{path}{exe}{'anvil-update-files'},
});
### NOTE: The timing is a guide only. The AJAX does a lot of work before this script is invoked. It
### might be better to just remove the timing stuff entirely...
my $size = (stat($out_file))[7];
@ -67,6 +75,8 @@ if ($cgi->param())
my $bytes_per_second = $anvil->Convert->round({number => ($size / $took), places => 0});
my $say_rate = $anvil->Words->string({key => "suffix_0001", variables => { number => $anvil->Convert->bytes_to_human_readable({'bytes' => $bytes_per_second}) }});
my $file_sum = $anvil->Get->md5sum({file => $out_file});
my $mimetype = mimetype($out_file);
my $executable = -x $out_file ? 1 : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
size => $size,
say_size_human => $say_size_human,
@ -76,7 +86,25 @@ if ($cgi->param())
bytes_per_second => $bytes_per_second,
say_rate => $say_rate,
file_sum => $file_sum,
mimetype => $mimetype,
executable => $executable,
}});
# Determine the type (guess) from the mimetype
my $ype = "other";
if ($mimetype =~ /cd-image/)
{
$type = "iso";
}
# This will need to be expanded over time
elsif (($executable) or (($mimetype =~ /perl/) or ($mimetype =~ /python/))
{
$type = "script";
}
elsif ($mimetype =~ /raw-disk-image/)
{
$type = "image";
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0260", variables => {
file => $out_file,
size_human => $say_size_human,
@ -85,6 +113,23 @@ if ($cgi->param())
took => $say_took,
md5sum => $file_sum
}});
# Try to connect to a database.
$anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
# If I have a database, record this file.
if ($anvil->data->{sys}{database}{connections})
{
# Add to files
my ($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2,
file_name => $file_name,
file_size => $size,
file_md5sum => $file_sum,
file_type => $file_type,
})
}
}
else
{

@ -1193,8 +1193,8 @@ ALTER TABLE file_locations OWNER TO admin;
CREATE TABLE history.file_locations (
history_id bigserial,
file_location_uuid uuid,
file_location_file_uuid text not null, -- This is
file_location_host_uuid text not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
file_location_file_uuid text,
file_location_host_uuid text,
modified_date timestamp with time zone not null
);
ALTER TABLE history.file_locations OWNER TO admin;
@ -1207,15 +1207,13 @@ BEGIN
SELECT INTO history_file_locations * FROM file_locations WHERE file_location_uuid = new.file_location_uuid;
INSERT INTO history.file_locations
(file_location_uuid,
file_location_name,
file_location_md5sum,
file_location_type,
file_location_file_uuid,
file_location_host_uuid,
modified_date)
VALUES
(history_file_locations.file_location_uuid,
history_file_locations.file_location_name,
history_file_locations.file_location_md5sum,
history_file_locations.file_location_type,
history_file_locations.file_location_file_uuid,
history_file_locations.file_location_host_uuid,
history_file_locations.modified_date);
RETURN NULL;
END;

@ -557,6 +557,7 @@ Finished Downloading: [#!variable!file!#].
</key>
<key name="log_0261">#!variable!file!# was called, but no files where available for download in CGI. Was the variable name 'upload_file' used?</key>
<key name="log_0262">[ Error ] - Storage->scan_directory() was asked to scan: [#!variable!directory!#], but it doesn't exist or isn't actually a directory.</key>
<key name="log_0263">Now deleting the file: [#!variable!file!#].</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>
@ -809,6 +810,12 @@ The update appears to have not completed successfully. The output was:
<key name="error_0046">This Striker system is not configured yet. This tool will not be available until it is.</key>
<key name="error_0047">Failed to start the Install Target feature. Got a non-zero return code when starting: [#!data!sys::daemon::dhcpd!#] (got: [#!variable!rc!#]).</key>
<key name="error_0048">Failed to stop the Install Target feature. Got a non-zero return code when starting: [#!data!sys::daemon::dhcpd!#] (got: [#!variable!rc!#]).</key>
<key name="error_0049">A request to rename a file was made, but no file name was given.</key>
<key name="error_0050">A request to rename the file: [#!variable!file!#] was made, but the new name wasn't given. Was '--to X' given?</key>
<key name="error_0051">A request to rename the file: [#!variable!file!#] was made, but that file doesn't exist.</key>
<key name="error_0051">A request to rename the file: [#!variable!file!#] to: [#!variable!to!#], but there is an existing file or directory with that name.</key>
<key name="error_0052">A request to delete a file was made, but no file name was given.</key>
<key name="error_0053">A request to delete the file: [#!variable!file!#] was received, but it is not under '/mnt/shared/'. This program can only work on or under that directory.</key>
<!-- These are units, words and so on used when displaying information. -->
<key name="unit_0001">Yes</key>

@ -13,12 +13,20 @@
# - If not found anywhere, remove it from 'file_locations' and send an alert.
# - If found, check the size. If it differs, recalculate the md5sum.
# - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage.
# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. If
# also used with '--everywhere', then all copies on all systems we know about will be deleted. This is
# done by registering a job against all known hosts. As such, if this is called and the target file
# doesn't exist, it just clears the job and then exits.
# - 5. If called with '--is-script=[0|1]', mark as 'script' in the 'files' table and set/remove the executable bit.
#
# Exit codes;
# 0 = Normal exit or md5sum of this program changed and it exited to reload.
# 1 =
# 2 =
# 1 = No databases available.
# 2 = Another process that is still running has picked up this job.
# 3 = No filename specified when needed by another switch.
# 4 = Request to change the file name but the new name wasn't given.
# 5 = The file specified was not found.
# 6 = The file to delete is not under '/mnt/shared/'.
#
# TODO:
# -
@ -45,9 +53,72 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new();
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'rename'} = "";
$anvil->data->{switches}{'is-script'} = "";
$anvil->data->{switches}{file} = "";
$anvil->data->{switches}{to} = "";
$anvil->data->{switches}{'delete'} = "";
$anvil->data->{switches}{everywhere} = "";
$anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"switches::job-uuid" => $anvil->data->{switches}{'job-uuid'},
"switches::rename" => $anvil->data->{switches}{'rename'},
"switches::is-script" => $anvil->data->{switches}{'is-script'},
"switches::file" => $anvil->data->{switches}{file},
"switches::to" => $anvil->data->{switches}{to},
"switches::delete" => $anvil->data->{switches}{'delete'},
"switches::everywhere" => $anvil->data->{switches}{everywhere},
}});
### When a new file is processed, record in DB and then update files.json (if on a striker)
# Connect or die
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
# If we have a job_uuid, pick it up.
if ($anvil->data->{switches}{'job-uuid'})
{
my $return = $anvil->Job->get_job_details({check => 1, job_uuid => $anvil->data->{switches}{'job-uuid'}});
if ($return == 1)
{
# It's not a UUID.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({code => 2});
}
if ($return == 2)
{
# This job is being handled by another process that is still alive.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({code => 3});
}
# If there's a progress, clear it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'job::job_progress' => $anvil->data->{job}{job_progress} }});
if ($anvil->data->{job}{job_progress})
{
$anvil->Job->update_progress({
progress => 1,
message => "clear",
job_uuid => $anvil->data->{jobs}{'job-uuid'},
});
}
}
# What are we doing?
if ($anvil->data->{switches}{'rename'})
{
handle_rename($anvil);
}
elsif ($anvil->data->{switches}{'delete'}))
{
handle_delete($anvil);
}
# We're done
$anvil->nice_exit({exit_code => 0});
@ -56,3 +127,84 @@ $anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Private functions. #
#############################################################################################################
# This handles deleting a file. If the requested deletion target doesn't exist, we'll just clear the
# database. If that doesn't exist either, we still don't error. This is to handle broad requests to delete a
# file everywhere. If we're asked to delete it everywhere, then we'll register a job against all hosts.
sub handle_delete
{
my ($anvil) = @_;
if (not $anvil->data->{switches}{file})
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052"});
$anvil->nice_exit({exit_code => 3});
}
elsif ($anvil->data->{switches}{file} !~ /^\/mnt\/shared\//)
{
# We don't do that here...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0053", variables => { file => $anvil->data->{switches}{file} }});
$anvil->nice_exit({exit_code => 6});
}
# Does the file exist?
if (-e $anvil->data->{switches}{file})
{
# Delete it.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0263", variables => { file => $anvil->data->{switches}{file} }});
unlink $anvil->data->{switches}{file};
# Sleep and verify
sleep 1;
if (-e $anvil->data->{switches}{file})
{
# Failed to delete...
# TODO:
}
else
{
# Deleted successfully. Remove from 'file_locations'
### TODO: Find the file_uuid, then if found, see if we have and entry in file_location and then delete it.
}
}
return(0);
}
# This handles renaming files.
sub handle_rename
{
my ($anvil) = @_;
# Do we have the current file name and the new one?
if (not $anvil->data->{switches}{file})
{
# Um...
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0049"});
$anvil->nice_exit({exit_code => 3});
}
elsif (not $anvil->data->{switches}{to})
{
# We need a target
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0050", variables => { file => $anvil->data->{switches}{file} }});
$anvil->nice_exit({exit_code => 4});
}
elsif (not -f $anvil->data->{switches}{file})
{
# The file to rename doesn't exist.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0051", variables => { file => $anvil->data->{switches}{file} }});
$anvil->nice_exit({exit_code => 5});
}
elsif (-e $anvil->data->{switches}{to})
{
# There's already a file (or directory or something) with that name.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0052", variables => { file => $anvil->data->{switches}{file}, to => $anvil->data->{switches}{to} }});
$anvil->nice_exit({exit_code => 6});
}
### TODO: Left off here
return(0);
}

@ -1,7 +1,6 @@
#!/usr/bin/perl
#
# This is the master daemon that manages all periodically run processes on Striker dashboards and Anvil!
# nodes.
# This updates things like the current network configuration, shared file data and writes it out to a json file.
#
#
use strict;

@ -50,8 +50,7 @@ $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure =
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
print $anvil->Words->string({key => "error_0003"})."\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0003"});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
@ -65,58 +64,14 @@ if (not $anvil->data->{switches}{'job-uuid'})
}
if ($anvil->data->{switches}{'job-uuid'})
{
# Is it set and valid?
if (not $anvil->Validate->is_uuid({uuid => $anvil->data->{switches}{'job-uuid'}}))
# Load the job details. If anything is returned, there was a problem.
my $return = $anvil->Job->get_job_details({job_uuid => $anvil->data->{switches}{'job-uuid'}});
if ($return)
{
# It's not a UUID.
print $anvil->Words->string({key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0033", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({code => 2});
}
# If I'm here, see if we can read the job details.
my $query = "
SELECT
job_host_uuid,
job_data,
job_updated,
job_name,
job_status
FROM
jobs
WHERE
job_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{switches}{'job-uuid'})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
if ($count < 1)
{
print $anvil->Words->string({key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} }})."\n";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0034", variables => { uuid => $anvil->data->{switches}{'job-uuid'} } });
$anvil->nice_exit({code => 2});
}
# If we're here, we're good. Load the details
$anvil->data->{jobs}{job_uuid} = $anvil->data->{switches}{'job-uuid'};
$anvil->data->{jobs}{job_host_uuid} = $results->[0]->[0];
$anvil->data->{jobs}{job_data} = $results->[0]->[1];
$anvil->data->{jobs}{job_updated} = $results->[0]->[2];
$anvil->data->{jobs}{job_name} = $results->[0]->[3];
$anvil->data->{jobs}{job_status} = $results->[0]->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
"jobs::job_host_uuid" => $anvil->data->{jobs}{job_host_uuid},
"jobs::job_data" => $anvil->data->{jobs}{job_data},
"jobs::job_updated" => $anvil->data->{jobs}{job_updated},
"jobs::job_name" => $anvil->data->{jobs}{job_name},
"jobs::job_status" => $anvil->data->{jobs}{job_status},
}});
}
# Clea any old runs.

@ -9,6 +9,7 @@
# 3 = Job was already picked up by another running instance.
# 4 = The host name did not update properly.
# 5 = Failed to write the temp file with the new password needed to call anvil-change-password.
# 6 = The job-uuid was not found.
#
use strict;
@ -749,73 +750,38 @@ sub pickup_job_details
{
my ($anvil) = @_;
# If any job said it was picked up, and the "job_picked_up_by" PID doesn't exist, take it and update
# it.
my $query = "
SELECT
job_uuid,
job_command,
job_data,
job_picked_up_by,
job_picked_up_at,
job_updated,
job_progress
FROM
jobs
WHERE
job_name = 'configure::network'
AND
job_progress != 100
AND
job_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->Get->host_uuid)."
LIMIT 1;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
# If this returns '1', the job-uuid was bad. If it returns '2', another process is running.
my $return = $anvil->Job->get_job_details({check => 1, job_uuid => $anvil->data->{switches}{'job-uuid'}});
if ($return == 1)
{
# It's not a valid UUID.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({code => 6});
}
if ($return == 2)
{
# This job is being handled by another process that is still alive.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({code => 3});
}
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
# Still alive? Good.
my $job_picked_up_by = $anvil->data->{jobs}{job_picked_up_by};
my $job_progress = $anvil->data->{jobs}{job_progress};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
results => $results,
count => $count,
}});
my $job_uuid = $results->[0]->[0];
my $job_command = $results->[0]->[1];
my $job_data = defined $results->[0]->[2] ? $results->[0]->[2] : "";
my $job_picked_up_by = $results->[0]->[3];
my $job_picked_up_at = $results->[0]->[4];
my $job_updated = $results->[0]->[5];
my $job_progress = $results->[0]->[6];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
job_uuid => $job_uuid,
job_command => $job_command,
job_data => $job_data,
job_picked_up_by => $job_picked_up_by,
job_picked_up_at => $job_picked_up_at,
job_updated => $job_updated,
job_progress => $job_progress,
}});
# See if the job was picked up by another running instance.
if ($job_picked_up_by)
{
# Check if the PID is still active.
$anvil->System->pids({ignore_me => 1});
# Is the PID that picked up the job still alive?
if (exists $anvil->data->{pids}{$job_picked_up_by})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0146", variables => { pid => $job_picked_up_by }});
$anvil->nice_exit({code => 3});
}
else
{
# The previous job is gone, we'll take this over.
# The previous job is gone if we're still alive, we'll take this over.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0147", variables => {
pid => $job_picked_up_by,
percent => $job_progress,
}});
}
}
# This will store the variables from the database
$anvil->data->{variables} = {};
@ -859,9 +825,6 @@ AND
$anvil->_make_hash_reference($anvil->data->{variables}, $this_variable, $this_value);
}
# This will be used when updating the job
$anvil->data->{job}{uuid} = $job_uuid;
# Record that we've picked up this job.
$anvil->Job->update_progress({
progress => 1,

Loading…
Cancel
Save