* Refined upload.pl a lot, moving it into Anvil::Tools. It also now handles file name collisions and reports basic info about the uploaded file.

* Updated Get->uuid to take the new 'short' parameter that, when passed, asks for just the first 8 bytes of the UUID string.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent b5378252fd
commit dd2b9ec026
  1. 3
      Anvil/Tools.pm
  2. 18
      Anvil/Tools/Get.pm
  3. 4
      cgi-bin/striker
  4. 118
      cgi-bin/upload.pl
  5. 30
      html/skins/alteeve/files.html
  6. 4
      share/anvil.sql
  7. 13
      share/words.xml
  8. 15
      tools/anvil-manage-files

@ -1032,6 +1032,9 @@ sub _set_paths
uuidgen => "/usr/bin/uuidgen",
virsh => "/usr/bin/virsh",
},
json => {
files => "files.json",
},
'lock' => {
database => "/tmp/anvil-tools.database.lock",
},

@ -907,7 +907,13 @@ sub users_home
=head2 uuid
This method returns a new v4 UUID (using 'UUID::Tiny'). It takes no parameters.
This method returns a new v4 UUID (using 'UUID::Tiny').
Parameters;
=head3 short (optional, default '0')
This returns just the first 8 bytes of the uuid. For example, if the generated UUID is C<< 9e4b3f7c-5a98-40b6-9c34-84fdb24ddd30 >>, only C<< 9e4b3f7c >> is returned.
=cut
sub uuid
@ -917,9 +923,19 @@ sub uuid
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
my $short = defined $parameter->{short} ? $parameter->{short} : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
short => $short,
}});
my $uuid = create_uuid_as_string(UUID_RANDOM);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
if ($short)
{
$uuid =~ s/^(\w+?)-.*$/$1/;
}
return($uuid);
}

@ -464,8 +464,8 @@ sub process_file_menu
{
my ($anvil) = @_;
$anvil->data->{form}{refresh_link} = "striker?anvil=true";
$anvil->data->{form}{back_link} = "?striker=true";
$anvil->data->{form}{refresh_link} = "striker?files=true";
#$anvil->data->{form}{back_link} = "?striker=true";
$anvil->data->{cgi}{task}{value} = "" if not defined $anvil->data->{cgi}{task}{value};
$anvil->data->{cgi}{action}{value} = "" if not defined $anvil->data->{cgi}{action}{value};

@ -1,73 +1,95 @@
#!/usr/bin/perl
#
# This is a special-purpose mini program used to handle upload requests. It has it's own micro-handling of
# CGI specifically set to grab data when triggered by Striker using the 'jQuery Upload File Plugin' from
# files.js.
#
#
use strict;
use warnings;
use CGI;
use Anvil::Tools;
my $cgi = CGI->new;
print q|Content-type: text/html; charset=utf-8
# Turn off buffering
$| = 1;
<!DOCTYPE html>
<html lang="en_CA">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<!-- Disable caching during development. -->
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" /> <title>Alteeve - Striker</title>
<link rel="stylesheet" href="/skins/alteeve/main.css" media="screen" />
<script type="text/javascript" src="/jquery-latest.js"></script>
<script type="text/javascript" src="/skins/alteeve/main.js"></script>
<!-- NOTE: These are for jquery-ui using the 'smoothness' skin. We may want to move this under the skin directory in case other skins want to use different jquery-ui skins someday. -->
<link rel="stylesheet" href="/jquery-ui-latest/jquery-ui.css">
<script type="text/javascript" src="/jquery-ui-latest/jquery-ui.js"></script>
<script type="text/javascript" src="/skins/alteeve/files.js"></script>
<link rel="stylesheet" href="/skins/alteeve/files.css">
|;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
my $lightweight_fh = $cgi->upload('field_name');
my $cgi = CGI->new;
print "Content-type: text/html; charset=utf-8\n\n";
print $anvil->Template->get({file => "files.html", name => "upload_header"})."\n";
my $lightweight_fh = $cgi->upload('field_name');
# undef may be returned if it's not a valid file handle
if ($cgi->param())
{
print q|
<title>Saving File...</title>
</head>
<body>
|;
my $start = time;
my $filename = $cgi->upload('upload_file');
my $out = "/mnt/shared/incoming/".$filename;
print "Saving file: [".$out."]\n";
my $out_file = $anvil->data->{path}{directories}{shared}{incoming}."/".$filename;
if (-e $out_file)
{
# Don't overwrite
$out_file .= "_".$anvil->Get->date_and_time({file_name => 1});
# If this exists (somehow), we'll append a short UUID
if (-e $out_file)
{
$out_file .= "_".$anvil->Get->uuid({short => 1});
}
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0259", variables => { file => $out_file }});
my $cgi_file_handle = $cgi->param('upload_file');
open(my $file_handle, ">$out") or die "failed to write: [$out], error: $!\n";
open(my $file_handle, ">$out_file") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0016", variables => { shell_call => $out_file, error => $! }});
while(<$cgi_file_handle>)
{
print $file_handle $_;
}
close $file_handle;
print "Done.\n";
### NOTE: The timing is a guide only. The AJAX does a lot of work before this script is invoked. It
### might be better to just remove the timing stuff entirely...
my $size = (stat($out_file))[7];
my $say_size_human = $anvil->Convert->add_commas({number => $size});
my $say_size_comma = $anvil->Convert->bytes_to_human_readable({'bytes' => $size});
my $took = time - $start;
$took = 1 if not $took;
my $say_took = $anvil->Convert->add_commas({number => $took});
my $bytes_per_second = $anvil->Convert->round({number => ($size / $took), places => 0});
my $say_rate = $anvil->Words->string({key => "suffix_0001", variables => { number => $anvil->Convert->bytes_to_human_readable({'bytes' => $bytes_per_second}) }});
my $file_sum = $anvil->Get->md5sum({file => $out_file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
size => $size,
say_size_human => $say_size_human,
say_size_comma => $say_size_comma,
took => $took,
say_took => $say_took,
bytes_per_second => $bytes_per_second,
say_rate => $say_rate,
file_sum => $file_sum,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0260", variables => {
file => $out_file,
size_human => $say_size_human,
size_bytes => $say_size_comma,
rate => $say_rate,
took => $say_took,
md5sum => $file_sum
}});
}
else
{
print q|
<title>Test Upload</title>
</head>
<body>
<h1>Upload file</h1>
<form method="post" enctype="multipart/form-data">
<!-- <input type="file" name="upload_file" value="Choose file"> -->
<!-- <input type="submit" name="submit" value="Upload"> -->
Upload
<div id="fileuploader">Upload</div>
</form>
|;
# Why are we here?
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "warn", key => "log_0261", variables => { file => $THIS_FILE }});
}
print "</body>\n";
exit(0);

@ -27,3 +27,33 @@
</form>
</table>
<!-- end main-menu -->
<!-- start upload_header -->
<!DOCTYPE html>
<html lang="en_CA">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<!-- Disable caching during development. -->
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" /> <title>Alteeve - Striker</title>
<link rel="stylesheet" href="/skins/alteeve/main.css" media="screen" />
<script type="text/javascript" src="/jquery-latest.js"></script>
<script type="text/javascript" src="/skins/alteeve/main.js"></script>
<!-- NOTE: These are for jquery-ui using the 'smoothness' skin. We may want to move this under the skin directory in case other skins want to use different jquery-ui skins someday. -->
<link rel="stylesheet" href="/jquery-ui-latest/jquery-ui.css">
<script type="text/javascript" src="/jquery-ui-latest/jquery-ui.js"></script>
<script type="text/javascript" src="/skins/alteeve/files.js"></script>
<link rel="stylesheet" href="/skins/alteeve/files.css">
<title>#!string!striker_0120!#</title>
</head>
<body>
#!string!striker_0120!#
</body>
<!-- end upload_header -->

@ -1102,6 +1102,7 @@ CREATE TRIGGER trigger_ip_addresses
CREATE TABLE files (
file_uuid uuid not null primary key,
file_name text not null, -- This is the file's name. It can change without re-uploading the file.
file_size numeric not null, -- This is the file's size in bytes. If it recorded as a quick way to determine if a file has changed on disk.
file_md5sum text not null, -- This is the sum as calculated when the file is first uploaded. Once recorded, it can't change.
file_type text not null, -- This is; 'iso', 'repo_rpm', 'script', or 'backup'.
modified_date timestamp with time zone not null
@ -1112,6 +1113,7 @@ CREATE TABLE history.files (
history_id bigserial,
file_uuid uuid,
file_name text,
file_size numeric,
file_md5sum text,
file_type text,
modified_date timestamp with time zone not null
@ -1127,12 +1129,14 @@ BEGIN
INSERT INTO history.files
(file_uuid,
file_name,
file_size,
file_md5sum,
file_type,
modified_date)
VALUES
(history_files.file_uuid,
history_files.file_name,
history_files.file_size,
history_files.file_md5sum,
history_files.file_type,
history_files.modified_date);

@ -547,6 +547,15 @@ The md5sum of: [#!variable!file!#] has changed since the daemon started.
<key name="log_0256">[ Note ] - Downloaded: [#!variable!file!#] (#!variable!human_readable_size!# / #!variable!size_in_bytes!# bytes).</key>
<key name="log_0257">[ Warning ] - It appears that we failed to downloaded and save: [#!variable!file!#].</key>
<key name="log_0258">[ Warning ] - It appears that we failed to downloaded and save: [#!variable!file!#]. The output file has no size, and will be removed.</key>
<key name="log_0259">Starting download of file: [#!variable!file!#].</key>
<key name="log_0260">
Finished Downloading: [#!variable!file!#].
- md5sum: ...... [#!variable!md5sum!#].
- Size: ........ [#!variable!size_human!# (#!variable!size_bytes!# bytes)].
- Took: ........ [#!variable!took!#] seconds.
- Download rate: [#!variable!rate!#]
</key>
<key name="log_0261">#!variable!file!# was called, but no files where available for download in CGI. Was the variable name 'upload_file' used?</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>
@ -705,6 +714,10 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st
<key name="striker_0117">Initial host configuration.</key>
<key name="striker_0118">Prepare a new machine for use as an Anvil! node or DR (disaster recovery) host. This process will setup the repository, install the appropriate anvil packages and link it to the Anvil! databases on the Strikers you choose.</key>
<key name="striker_0119">Anvil! File Manager.</key>
<key name="striker_0120">Saving File...</key>
<!-- These are generally units and appended to numbers -->
<key name="suffix_0001">#!variable!number!#/sec</key>
<!-- Strings used by jobs -->
<key name="job_0001">Configure Network</key>

@ -1,6 +1,19 @@
#!/usr/bin/perl
#
# This handles moving around and managing files on Anvil! nodes, DR hosts and Striker dashboards.
#
# When this is called (periodically by the daemon of after an upload / ISO generation);
# - 1. Check 'incoming/' for files. For any found, generate an md5sum and see if the file name and sum match
# anything in the database from any host;
# - If so, update/add an entry in 'file_locations'
# - If not, create a new entry in 'files' and then add the first entry in 'file_locations'
# - 2. Check 'file_locations' for any files on this system, and verify they exist still.
# - If not, check the other files for one with a matching md5sum. If found, handle as a rename.
# - If not found at all, search for the file according to the search rules and copy to here if found.
# - If not found anywhere, remove it from 'file_locations' and send an alert.
# - If found, check the size. If it differs, recalculate the md5sum.
# - 3. If called with '--rename --file <filename> --to <newname>', rename the file and update 'files'.
# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage.
#
# Exit codes;
# 0 = Normal exit or md5sum of this program changed and it exited to reload.
@ -33,6 +46,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new();
### When a new file is processed, record in DB and then update files.json (if on a striker)
# We're done
$anvil->nice_exit({exit_code => 0});

Loading…
Cancel
Save