From 47280c8fd9a7d0423cf8e17ffde67a949a2bc8b4 Mon Sep 17 00:00:00 2001 From: Digimer Date: Fri, 22 Feb 2019 02:10:55 -0500 Subject: [PATCH] * Added 'files -> file_directory' to help distinquish cases where two different files of the same name exist in different directories. Signed-off-by: Digimer --- Anvil/Tools/Database.pm | 95 +++++++++++++++---------- share/anvil.sql | 4 ++ tools/anvil-manage-files | 150 +++++++++++++++++++++++---------------- 3 files changed, 153 insertions(+), 96 deletions(-) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 2e0b0d9c..f41dd707 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -2389,6 +2389,10 @@ If not passed, a check will be made to see if an existing entry is found for C<< This is the file's name. +=head3 file_directory (required) + +This is the directory that the file is in. This is used to avoid conflict if two files of the same name exist in two places but are otherwise different. + =head3 file_size (required) This is the file's size in bytes. It is recorded as a quick way to determine if the file has changed on disk. @@ -2414,21 +2418,23 @@ sub insert_or_update_files my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }}); - my $file = defined $parameter->{file} ? $parameter->{file} : ""; - my $line = defined $parameter->{line} ? $parameter->{line} : ""; - my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : ""; - my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : ""; - my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : ""; - my $file_md5sum = defined $parameter->{file_md5sum} ? $parameter->{file_md5sum} : ""; - my $file_type = defined $parameter->{file_type} ? $parameter->{file_type} : ""; - my $file_mtime = defined $parameter->{file_mtime} ? $parameter->{file_mtime} : ""; + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $line = defined $parameter->{line} ? $parameter->{line} : ""; + my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : ""; + my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : ""; + my $file_directory = defined $parameter->{file_directory} ? $parameter->{file_directory} : ""; + my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : ""; + my $file_md5sum = defined $parameter->{file_md5sum} ? $parameter->{file_md5sum} : ""; + my $file_type = defined $parameter->{file_type} ? $parameter->{file_type} : ""; + my $file_mtime = defined $parameter->{file_mtime} ? $parameter->{file_mtime} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - file_uuid => $file_uuid, - file_name => $file_name, - file_size => $file_size, - file_md5sum => $file_md5sum, - file_type => $file_type, - file_mtime => $file_mtime, + file_uuid => $file_uuid, + file_name => $file_name, + file_directory => $file_directory, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_type => $file_type, + file_mtime => $file_mtime, }}); if (not $file_name) @@ -2443,6 +2449,12 @@ sub insert_or_update_files $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_name" }}); return(""); } + if (not $file_directory) + { + # Throw an error and exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_directory" }}); + return(""); + } if (not $file_md5sum) { # Throw an error and exit. @@ -2524,6 +2536,7 @@ INSERT INTO ( file_uuid, file_name, + file_directory, file_size, file_md5sum, file_type, @@ -2532,6 +2545,7 @@ INSERT INTO ) VALUES ( ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).", + ".$anvil->data->{sys}{database}{use_handle}->quote($file_directory).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).", @@ -2548,6 +2562,7 @@ INSERT INTO my $query = " SELECT file_name, + file_directory, file_size, file_md5sum, file_type, @@ -2573,39 +2588,43 @@ WHERE } foreach my $row (@{$results}) { - my $old_file_name = $row->[0]; - my $old_file_size = $row->[1]; - my $old_file_md5sum = $row->[2]; - my $old_file_type = $row->[3]; - my $old_file_mtime = $row->[4]; + my $old_file_name = $row->[0]; + my $old_file_directory = $row->[1]; + my $old_file_size = $row->[1]; + my $old_file_md5sum = $row->[2]; + my $old_file_type = $row->[3]; + my $old_file_mtime = $row->[4]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - old_file_name => $old_file_name, - old_file_size => $old_file_size, - old_file_md5sum => $old_file_md5sum, - old_file_type => $old_file_type, - old_file_mtime => $old_file_mtime, + old_file_name => $old_file_name, + old_file_directory => $old_file_directory, + old_file_size => $old_file_size, + old_file_md5sum => $old_file_md5sum, + old_file_type => $old_file_type, + old_file_mtime => $old_file_mtime, }}); # Anything change? - if (($old_file_name ne $file_name) or - ($old_file_size ne $file_size) or - ($old_file_md5sum ne $file_md5sum) or - ($old_file_mtime ne $file_mtime) or - ($old_file_type ne $file_type)) + if (($old_file_name ne $file_name) or + ($old_file_directory ne $file_directory) or + ($old_file_size ne $file_size) or + ($old_file_md5sum ne $file_md5sum) or + ($old_file_mtime ne $file_mtime) or + ($old_file_type ne $file_type)) { # Something changed, save. my $query = " UPDATE files SET - file_name = ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).", - file_size = ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).", - file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).", - file_type = ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).", - file_mtime = ".$anvil->data->{sys}{database}{use_handle}->quote($file_mtime).", - modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})." + file_name = ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).", + file_directory = ".$anvil->data->{sys}{database}{use_handle}->quote($file_directory).", + file_size = ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).", + file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).", + file_type = ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).", + file_mtime = ".$anvil->data->{sys}{database}{use_handle}->quote($file_mtime).", + modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})." WHERE - file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)." + file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)." "; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); @@ -5347,6 +5366,9 @@ sub query $anvil->Database->check_lock_age({debug => $debug}); # Do I need to log the transaction? + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "sys::database::log_transactions" => $anvil->data->{sys}{database}{log_transactions}, + }}); if ($anvil->data->{sys}{database}{log_transactions}) { $anvil->Log->entry({source => $source, line => $line, secure => $secure, level => 0, key => "log_0074", variables => { @@ -5361,6 +5383,7 @@ sub query server => $say_server, db_error => $DBI::errstr, }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { DBreq => $DBreq }}); # Execute on the query $DBreq->execute() or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0076", variables => { diff --git a/share/anvil.sql b/share/anvil.sql index 4c6f737d..38cc906d 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -1102,6 +1102,7 @@ CREATE TRIGGER trigger_ip_addresses CREATE TABLE files ( file_uuid uuid not null primary key, file_name text not null, -- This is the file's name. It can change without re-uploading the file. + file_directory text not null, -- This is the directory that the file is in. file_size numeric not null, -- This is the file's size in bytes. If it recorded as a quick way to determine if a file has changed on disk. file_md5sum text not null, -- This is the sum as calculated when the file is first uploaded. Once recorded, it can't change. file_type text not null, -- This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 'other'. @@ -1114,6 +1115,7 @@ CREATE TABLE history.files ( history_id bigserial, file_uuid uuid, file_name text, + file_directory text, file_size numeric, file_md5sum text, file_type text, @@ -1131,6 +1133,7 @@ BEGIN INSERT INTO history.files (file_uuid, file_name, + file_directory, file_size, file_md5sum, file_type, @@ -1139,6 +1142,7 @@ BEGIN VALUES (history_files.file_uuid, history_files.file_name, + history_files.file_directory, history_files.file_size, history_files.file_md5sum, history_files.file_type, diff --git a/tools/anvil-manage-files b/tools/anvil-manage-files index aaeaf742..27ed584d 100755 --- a/tools/anvil-manage-files +++ b/tools/anvil-manage-files @@ -40,7 +40,6 @@ use warnings; use Anvil::Tools; use Data::Dumper; - # Disable buffering $| = 1; @@ -167,33 +166,43 @@ sub find_missing_files my ($anvil) = @_; # What am I? This will impact how missing files are found. -# my $query = " -# SELECT -# file_location_file_uuid -# FROM -# file_locations -# WHERE -# file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})." -# ;"; -# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); -# -# my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); -# my $count = @{$results}; -# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { -# results => $results, -# count => $count, -# }}); -# foreach my $row (@{$results}) -# { -# my $file_location_file_uuid = $row->[0]; -# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { -# file_location_file_uuid => $file_location_file_uuid, -# }}); -# -# ### TODO: How to handle when the file with the same name exists on 2+ machines with -# ### different md5sums. Use the most recent mtime? -# # Read in the file details. -# } + my $query = " +SELECT + a.file_uuid, + a.file_name +FROM + files a, + hosts b, + file_locations c +WHERE + b.host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})." +AND + a.file_uuid = c.file_location_file_uuid +AND + b.host_uuid = c.file_location_host_uuid +ORDER BY + a.file_name ASC, + b.host_name ASC +;"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); + my $count = @{$results}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + results => $results, + count => $count, + }}); + foreach my $row (@{$results}) + { + my $file_location_file_uuid = $row->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + file_location_file_uuid => $file_location_file_uuid, + }}); + + ### TODO: How to handle when the file with the same name exists on 2+ machines with + ### different md5sums. Use the most recent mtime? + # Read in the file details. + } # Read in any entries from 'file_locations'. @@ -212,7 +221,7 @@ sub check_incoming } # Read any files in '/mnt/shared/incoming'. $anvil->Storage->scan_directory({ - debug => 2, + debug => 1, directory => $anvil->data->{path}{directories}{shared}{base}, recursive => 1, }); @@ -230,6 +239,7 @@ sub check_incoming next if $file_type ne "file"; my $file_name = $anvil->data->{scan}{directories}{$full_path}{name}; + my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory}; my $file_size = $anvil->data->{scan}{directories}{$full_path}{size}; my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime}; my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; @@ -287,13 +297,14 @@ sub check_incoming # Insert or update the files entry. ($file_uuid) = $anvil->Database->insert_or_update_files({ - debug => 2, - file_uuid => $file_uuid, - file_name => $file_name, - file_size => $file_size, - file_md5sum => $file_md5sum, - file_mtime => $file_mtime, - file_type => $say_mimetype, + debug => 2, + file_uuid => $file_uuid, + file_name => $file_name, + file_directory => $file_directory, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_mtime => $file_mtime, + file_type => $say_mimetype, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); } @@ -334,6 +345,20 @@ sub check_incoming source_file => $full_path, target_file => $target, }); + + # Update the file_directory. + my $say_directory =~ s/\/$//; + ($file_uuid) = $anvil->Database->insert_or_update_files({ + debug => 2, + file_uuid => $file_uuid, + file_name => $file_name, + file_directory => $say_directory, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_mtime => $file_mtime, + file_type => $say_mimetype, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); } } @@ -476,6 +501,7 @@ sub handle_script my $query = " SELECT file_name, + file_directory, file_size, file_md5sum, file_type, @@ -489,17 +515,19 @@ WHERE my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }}); - my $file_name = $results->[0]->[0]; - my $file_size = $results->[0]->[1]; - my $file_md5sum = $results->[0]->[2]; - my $file_type = $results->[0]->[3]; - my $file_mtime = $results->[0]->[4]; + my $file_name = $results->[0]->[0]; + my $file_directory = $results->[0]->[1]; + my $file_size = $results->[0]->[2]; + my $file_md5sum = $results->[0]->[3]; + my $file_type = $results->[0]->[4]; + my $file_mtime = $results->[0]->[5]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - file_name => $file_name, - file_size => $file_size, - file_md5sum => $file_md5sum, - file_type => $file_type, - file_mtime => $file_mtime, + file_name => $file_name, + file_directory => $file_directory, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_type => $file_type, + file_mtime => $file_mtime, }}); if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'})) { @@ -510,26 +538,28 @@ WHERE my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }}); $anvil->Database->insert_or_update_files({ - debug => 2, - file_uuid => $file_uuid, - file_name => $anvil->data->{switches}{file}, - file_size => $file_size, - file_md5sum => $file_md5sum, - file_mtime => $file_mtime, - file_type => $say_mimetype, + debug => 2, + file_uuid => $file_uuid, + file_name => $anvil->data->{switches}{file}, + file_directory => $file_directory, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_mtime => $file_mtime, + file_type => $say_mimetype, }); } elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'})) { # Change the file tpye to "script" $anvil->Database->insert_or_update_files({ - debug => 2, - file_uuid => $file_uuid, - file_name => $anvil->data->{switches}{file}, - file_size => $file_size, - file_md5sum => $file_md5sum, - file_mtime => $file_mtime, - file_type => "script", + debug => 2, + file_uuid => $file_uuid, + file_name => $anvil->data->{switches}{file}, + file_directory => $file_directory, + file_size => $file_size, + file_md5sum => $file_md5sum, + file_mtime => $file_mtime, + file_type => "script", }); } }