diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index 13509c7f..93891e55 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1032,6 +1032,9 @@ sub _set_paths uuidgen => "/usr/bin/uuidgen", virsh => "/usr/bin/virsh", }, + json => { + files => "files.json", + }, 'lock' => { database => "/tmp/anvil-tools.database.lock", }, diff --git a/Anvil/Tools/Get.pm b/Anvil/Tools/Get.pm index 67035122..ebb922fa 100644 --- a/Anvil/Tools/Get.pm +++ b/Anvil/Tools/Get.pm @@ -907,7 +907,13 @@ sub users_home =head2 uuid -This method returns a new v4 UUID (using 'UUID::Tiny'). It takes no parameters. +This method returns a new v4 UUID (using 'UUID::Tiny'). + +Parameters; + +=head3 short (optional, default '0') + +This returns just the first 8 bytes of the uuid. For example, if the generated UUID is C<< 9e4b3f7c-5a98-40b6-9c34-84fdb24ddd30 >>, only C<< 9e4b3f7c >> is returned. =cut sub uuid @@ -917,9 +923,19 @@ sub uuid my $anvil = $self->parent; my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + my $short = defined $parameter->{short} ? $parameter->{short} : 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + short => $short, + }}); + my $uuid = create_uuid_as_string(UUID_RANDOM); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }}); + if ($short) + { + $uuid =~ s/^(\w+?)-.*$/$1/; + } + return($uuid); } diff --git a/cgi-bin/striker b/cgi-bin/striker index 594a153d..a4dfb954 100755 --- a/cgi-bin/striker +++ b/cgi-bin/striker @@ -464,8 +464,8 @@ sub process_file_menu { my ($anvil) = @_; - $anvil->data->{form}{refresh_link} = "striker?anvil=true"; - $anvil->data->{form}{back_link} = "?striker=true"; + $anvil->data->{form}{refresh_link} = "striker?files=true"; + #$anvil->data->{form}{back_link} = "?striker=true"; $anvil->data->{cgi}{task}{value} = "" if not defined $anvil->data->{cgi}{task}{value}; $anvil->data->{cgi}{action}{value} = "" if not defined $anvil->data->{cgi}{action}{value}; diff --git a/cgi-bin/upload.pl b/cgi-bin/upload.pl index a324744b..36499cef 100755 --- a/cgi-bin/upload.pl +++ b/cgi-bin/upload.pl @@ -1,73 +1,95 @@ #!/usr/bin/perl +# +# This is a special-purpose mini program used to handle upload requests. It has it's own micro-handling of +# CGI specifically set to grab data when triggered by Striker using the 'jQuery Upload File Plugin' from +# files.js. +# +# use strict; use warnings; use CGI; +use Anvil::Tools; -my $cgi = CGI->new; -print q|Content-type: text/html; charset=utf-8 +# Turn off buffering +$| = 1; - - - - - - - - - - - Alteeve - Striker - - - - - - - - - - - -|; +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} +my $anvil = Anvil::Tools->new(); +$anvil->Log->level({set => 2}); -my $lightweight_fh = $cgi->upload('field_name'); +my $cgi = CGI->new; + +print "Content-type: text/html; charset=utf-8\n\n"; +print $anvil->Template->get({file => "files.html", name => "upload_header"})."\n"; + +my $lightweight_fh = $cgi->upload('field_name'); # undef may be returned if it's not a valid file handle if ($cgi->param()) { - print q| - Saving File... - - -|; + my $start = time; my $filename = $cgi->upload('upload_file'); - my $out = "/mnt/shared/incoming/".$filename; - print "Saving file: [".$out."]\n"; + my $out_file = $anvil->data->{path}{directories}{shared}{incoming}."/".$filename; + if (-e $out_file) + { + # Don't overwrite + $out_file .= "_".$anvil->Get->date_and_time({file_name => 1}); + + # If this exists (somehow), we'll append a short UUID + if (-e $out_file) + { + $out_file .= "_".$anvil->Get->uuid({short => 1}); + } + } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0259", variables => { file => $out_file }}); my $cgi_file_handle = $cgi->param('upload_file'); - open(my $file_handle, ">$out") or die "failed to write: [$out], error: $!\n"; + open(my $file_handle, ">$out_file") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0016", variables => { shell_call => $out_file, error => $! }}); while(<$cgi_file_handle>) { print $file_handle $_; } close $file_handle; - print "Done.\n"; + + ### NOTE: The timing is a guide only. The AJAX does a lot of work before this script is invoked. It + ### might be better to just remove the timing stuff entirely... + my $size = (stat($out_file))[7]; + my $say_size_human = $anvil->Convert->add_commas({number => $size}); + my $say_size_comma = $anvil->Convert->bytes_to_human_readable({'bytes' => $size}); + my $took = time - $start; + $took = 1 if not $took; + my $say_took = $anvil->Convert->add_commas({number => $took}); + my $bytes_per_second = $anvil->Convert->round({number => ($size / $took), places => 0}); + my $say_rate = $anvil->Words->string({key => "suffix_0001", variables => { number => $anvil->Convert->bytes_to_human_readable({'bytes' => $bytes_per_second}) }}); + my $file_sum = $anvil->Get->md5sum({file => $out_file}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + size => $size, + say_size_human => $say_size_human, + say_size_comma => $say_size_comma, + took => $took, + say_took => $say_took, + bytes_per_second => $bytes_per_second, + say_rate => $say_rate, + file_sum => $file_sum, + }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0260", variables => { + file => $out_file, + size_human => $say_size_human, + size_bytes => $say_size_comma, + rate => $say_rate, + took => $say_took, + md5sum => $file_sum + }}); } else { - print q| - Test Upload - - -

Upload file

-
- - - Upload -
Upload
-
-|; + # Why are we here? + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "warn", key => "log_0261", variables => { file => $THIS_FILE }}); } -print "\n"; exit(0); diff --git a/html/skins/alteeve/files.html b/html/skins/alteeve/files.html index e6d5cb74..176249cf 100644 --- a/html/skins/alteeve/files.html +++ b/html/skins/alteeve/files.html @@ -27,3 +27,33 @@ + + + + + + + + + + + + + Alteeve - Striker + + + + + + + + + + + + #!string!striker_0120!# + + +#!string!striker_0120!# + + diff --git a/share/anvil.sql b/share/anvil.sql index b6fce000..f671ef20 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -1102,6 +1102,7 @@ CREATE TRIGGER trigger_ip_addresses CREATE TABLE files ( file_uuid uuid not null primary key, file_name text not null, -- This is the file's name. It can change without re-uploading the file. + file_size numeric not null, -- This is the file's size in bytes. If it recorded as a quick way to determine if a file has changed on disk. file_md5sum text not null, -- This is the sum as calculated when the file is first uploaded. Once recorded, it can't change. file_type text not null, -- This is; 'iso', 'repo_rpm', 'script', or 'backup'. modified_date timestamp with time zone not null @@ -1112,6 +1113,7 @@ CREATE TABLE history.files ( history_id bigserial, file_uuid uuid, file_name text, + file_size numeric, file_md5sum text, file_type text, modified_date timestamp with time zone not null @@ -1127,12 +1129,14 @@ BEGIN INSERT INTO history.files (file_uuid, file_name, + file_size, file_md5sum, file_type, modified_date) VALUES (history_files.file_uuid, history_files.file_name, + history_files.file_size, history_files.file_md5sum, history_files.file_type, history_files.modified_date); diff --git a/share/words.xml b/share/words.xml index 5df6e70c..e42bea58 100644 --- a/share/words.xml +++ b/share/words.xml @@ -547,6 +547,15 @@ The md5sum of: [#!variable!file!#] has changed since the daemon started. [ Note ] - Downloaded: [#!variable!file!#] (#!variable!human_readable_size!# / #!variable!size_in_bytes!# bytes). [ Warning ] - It appears that we failed to downloaded and save: [#!variable!file!#]. [ Warning ] - It appears that we failed to downloaded and save: [#!variable!file!#]. The output file has no size, and will be removed. + Starting download of file: [#!variable!file!#]. + +Finished Downloading: [#!variable!file!#]. +- md5sum: ...... [#!variable!md5sum!#]. +- Size: ........ [#!variable!size_human!# (#!variable!size_bytes!# bytes)]. +- Took: ........ [#!variable!took!#] seconds. +- Download rate: [#!variable!rate!#] + + #!variable!file!# was called, but no files where available for download in CGI. Was the variable name 'upload_file' used? Test @@ -705,6 +714,10 @@ Here we will inject 't_0006', which injects 't_0001' which has a variable: [#!st Initial host configuration. Prepare a new machine for use as an Anvil! node or DR (disaster recovery) host. This process will setup the repository, install the appropriate anvil packages and link it to the Anvil! databases on the Strikers you choose. Anvil! File Manager. + Saving File... + + + #!variable!number!#/sec Configure Network diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index ce30f2f6..88b737bd 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -1,6 +1,19 @@ #!/usr/bin/perl # # This handles moving around and managing files on Anvil! nodes, DR hosts and Striker dashboards. +# +# When this is called (periodically by the daemon of after an upload / ISO generation); +# - 1. Check 'incoming/' for files. For any found, generate an md5sum and see if the file name and sum match +# anything in the database from any host; +# - If so, update/add an entry in 'file_locations' +# - If not, create a new entry in 'files' and then add the first entry in 'file_locations' +# - 2. Check 'file_locations' for any files on this system, and verify they exist still. +# - If not, check the other files for one with a matching md5sum. If found, handle as a rename. +# - If not found at all, search for the file according to the search rules and copy to here if found. +# - If not found anywhere, remove it from 'file_locations' and send an alert. +# - If found, check the size. If it differs, recalculate the md5sum. +# - 3. If called with '--rename --file --to ', rename the file and update 'files'. +# - 4. If called with '--delete', remove from 'file_locations' and then remove from the local storage. # # Exit codes; # 0 = Normal exit or md5sum of this program changed and it exited to reload. @@ -33,6 +46,8 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) my $anvil = Anvil::Tools->new(); +### When a new file is processed, record in DB and then update files.json (if on a striker) + # We're done $anvil->nice_exit({exit_code => 0});