seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Added 'files -> file_directory' to help distinquish cases where two different files of the same name exist in different directories.

Browse Source 
Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent 529c12d2e2
commit 47280c8fd9
3 changed files with 153 additions and 96 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 95
      Anvil/Tools/Database.pm
  2. 4
      share/anvil.sql
  3. 150
      tools/anvil-manage-files

95
Anvil/Tools/Database.pm
Unescape View File

@ -2389,6 +2389,10 @@ If not passed, a check will be made to see if an existing entry is found for C<<
This is the file's name. This is the file's name.
=head3 file_directory (required)
This is the directory that the file is in. This is used to avoid conflict if two files of the same name exist in two places but are otherwise different.
=head3 file_size (required) =head3 file_size (required)
This is the file's size in bytes. It is recorded as a quick way to determine if the file has changed on disk. This is the file's size in bytes. It is recorded as a quick way to determine if the file has changed on disk.
@ -2414,21 +2418,23 @@ sub insert_or_update_files
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }});
my $file = defined $parameter->{file} ? $parameter->{file} : ""; my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $line = defined $parameter->{line} ? $parameter->{line} : ""; my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : ""; my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : "";
my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : ""; my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : "";
my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : ""; my $file_directory = defined $parameter->{file_directory} ? $parameter->{file_directory} : "";
my $file_md5sum = defined $parameter->{file_md5sum} ? $parameter->{file_md5sum} : ""; my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : "";
my $file_type = defined $parameter->{file_type} ? $parameter->{file_type} : ""; my $file_md5sum = defined $parameter->{file_md5sum} ? $parameter->{file_md5sum} : "";
my $file_mtime = defined $parameter->{file_mtime} ? $parameter->{file_mtime} : ""; my $file_type = defined $parameter->{file_type} ? $parameter->{file_type} : "";
my $file_mtime = defined $parameter->{file_mtime} ? $parameter->{file_mtime} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_name => $file_name, file_name => $file_name,
file_size => $file_size, file_directory => $file_directory,
file_md5sum => $file_md5sum, file_size => $file_size,
file_type => $file_type, file_md5sum => $file_md5sum,
file_mtime => $file_mtime, file_type => $file_type,
file_mtime => $file_mtime,
}}); }});
if (not $file_name) if (not $file_name)
@ -2443,6 +2449,12 @@ sub insert_or_update_files
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_name" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_name" }});
return(""); return("");
} }
if (not $file_directory)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_directory" }});
return("");
}
if (not $file_md5sum) if (not $file_md5sum)
{ {
# Throw an error and exit. # Throw an error and exit.
@ -2524,6 +2536,7 @@ INSERT INTO
( (
file_uuid, file_uuid,
file_name, file_name,
file_directory,
file_size, file_size,
file_md5sum, file_md5sum,
file_type, file_type,
@ -2532,6 +2545,7 @@ INSERT INTO
) VALUES ( ) VALUES (
".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_name).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_directory).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_size).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_type).", ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
@ -2548,6 +2562,7 @@ INSERT INTO
my $query = " my $query = "
SELECT SELECT
file_name, file_name,
file_directory,
file_size, file_size,
file_md5sum, file_md5sum,
file_type, file_type,
@ -2573,39 +2588,43 @@ WHERE
} }
foreach my $row (@{$results}) foreach my $row (@{$results})
{ {
my $old_file_name = $row->[0]; my $old_file_name = $row->[0];
my $old_file_size = $row->[1]; my $old_file_directory = $row->[1];
my $old_file_md5sum = $row->[2]; my $old_file_size = $row->[1];
my $old_file_type = $row->[3]; my $old_file_md5sum = $row->[2];
my $old_file_mtime = $row->[4]; my $old_file_type = $row->[3];
my $old_file_mtime = $row->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
old_file_name => $old_file_name, old_file_name => $old_file_name,
old_file_size => $old_file_size, old_file_directory => $old_file_directory,
old_file_md5sum => $old_file_md5sum, old_file_size => $old_file_size,
old_file_type => $old_file_type, old_file_md5sum => $old_file_md5sum,
old_file_mtime => $old_file_mtime, old_file_type => $old_file_type,
old_file_mtime => $old_file_mtime,
}}); }});
# Anything change? # Anything change?
if (($old_file_name ne $file_name) or if (($old_file_name ne $file_name) or
($old_file_size ne $file_size) or ($old_file_directory ne $file_directory) or
($old_file_md5sum ne $file_md5sum) or ($old_file_size ne $file_size) or
($old_file_mtime ne $file_mtime) or ($old_file_md5sum ne $file_md5sum) or
($old_file_type ne $file_type)) ($old_file_mtime ne $file_mtime) or
($old_file_type ne $file_type))
{ {
# Something changed, save. # Something changed, save.
my $query = " my $query = "
UPDATE UPDATE
files files
SET SET
file_name = ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).", file_name = ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).",
file_size = ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).", file_directory = ".$anvil->data->{sys}{database}{use_handle}->quote($file_directory).",
file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).", file_size = ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).",
file_type = ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).", file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
file_mtime = ".$anvil->data->{sys}{database}{use_handle}->quote($file_mtime).", file_type = ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})." file_mtime = ".$anvil->data->{sys}{database}{use_handle}->quote($file_mtime).",
modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
WHERE WHERE
file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)." file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)."
"; ";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); $anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
@ -5347,6 +5366,9 @@ sub query
$anvil->Database->check_lock_age({debug => $debug}); $anvil->Database->check_lock_age({debug => $debug});
# Do I need to log the transaction? # Do I need to log the transaction?
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::log_transactions" => $anvil->data->{sys}{database}{log_transactions},
}});
if ($anvil->data->{sys}{database}{log_transactions}) if ($anvil->data->{sys}{database}{log_transactions})
{ {
$anvil->Log->entry({source => $source, line => $line, secure => $secure, level => 0, key => "log_0074", variables => { $anvil->Log->entry({source => $source, line => $line, secure => $secure, level => 0, key => "log_0074", variables => {
@ -5361,6 +5383,7 @@ sub query
server => $say_server, server => $say_server,
db_error => $DBI::errstr, db_error => $DBI::errstr,
}}); }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { DBreq => $DBreq }});
# Execute on the query # Execute on the query
$DBreq->execute() or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0076", variables => { $DBreq->execute() or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0076", variables => {

4
share/anvil.sql
Unescape View File

@ -1102,6 +1102,7 @@ CREATE TRIGGER trigger_ip_addresses
CREATE TABLE files ( CREATE TABLE files (
file_uuid uuid not null primary key, file_uuid uuid not null primary key,
file_name text not null, -- This is the file's name. It can change without re-uploading the file. file_name text not null, -- This is the file's name. It can change without re-uploading the file.
file_directory text not null, -- This is the directory that the file is in.
file_size numeric not null, -- This is the file's size in bytes. If it recorded as a quick way to determine if a file has changed on disk. file_size numeric not null, -- This is the file's size in bytes. If it recorded as a quick way to determine if a file has changed on disk.
file_md5sum text not null, -- This is the sum as calculated when the file is first uploaded. Once recorded, it can't change. file_md5sum text not null, -- This is the sum as calculated when the file is first uploaded. Once recorded, it can't change.
file_type text not null, -- This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 'other'. file_type text not null, -- This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 'other'.
@ -1114,6 +1115,7 @@ CREATE TABLE history.files (
history_id bigserial, history_id bigserial,
file_uuid uuid, file_uuid uuid,
file_name text, file_name text,
file_directory text,
file_size numeric, file_size numeric,
file_md5sum text, file_md5sum text,
file_type text, file_type text,
@ -1131,6 +1133,7 @@ BEGIN
INSERT INTO history.files INSERT INTO history.files
(file_uuid, (file_uuid,
file_name, file_name,
file_directory,
file_size, file_size,
file_md5sum, file_md5sum,
file_type, file_type,
@ -1139,6 +1142,7 @@ BEGIN
VALUES VALUES
(history_files.file_uuid, (history_files.file_uuid,
history_files.file_name, history_files.file_name,
history_files.file_directory,
history_files.file_size, history_files.file_size,
history_files.file_md5sum, history_files.file_md5sum,
history_files.file_type, history_files.file_type,

150
tools/anvil-manage-files
Unescape View File

@ -40,7 +40,6 @@ use warnings;
use Anvil::Tools; use Anvil::Tools;
use Data::Dumper; use Data::Dumper;
# Disable buffering # Disable buffering
$| = 1; $| = 1;
@ -167,33 +166,43 @@ sub find_missing_files
my ($anvil) = @_; my ($anvil) = @_;
# What am I? This will impact how missing files are found. # What am I? This will impact how missing files are found.
# my $query = " my $query = "
# SELECT SELECT
# file_location_file_uuid a.file_uuid,
# FROM a.file_name
# file_locations FROM
# WHERE files a,
# file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})." hosts b,
# ;"; file_locations c
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); WHERE
# b.host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})."
# my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); AND
# my $count = @{$results}; a.file_uuid = c.file_location_file_uuid
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { AND
# results => $results, b.host_uuid = c.file_location_host_uuid
# count => $count, ORDER BY
# }}); a.file_name ASC,
# foreach my $row (@{$results}) b.host_name ASC
# { ;";
# my $file_location_file_uuid = $row->[0]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
# file_location_file_uuid => $file_location_file_uuid, my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
# }}); my $count = @{$results};
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
# ### TODO: How to handle when the file with the same name exists on 2+ machines with results => $results,
# ### different md5sums. Use the most recent mtime? count => $count,
# # Read in the file details. }});
# } foreach my $row (@{$results})
{
my $file_location_file_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_location_file_uuid => $file_location_file_uuid,
}});
### TODO: How to handle when the file with the same name exists on 2+ machines with
### different md5sums. Use the most recent mtime?
# Read in the file details.
}
# Read in any entries from 'file_locations'. # Read in any entries from 'file_locations'.
@ -212,7 +221,7 @@ sub check_incoming
} }
# Read any files in '/mnt/shared/incoming'. # Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({ $anvil->Storage->scan_directory({
debug => 2, debug => 1,
directory => $anvil->data->{path}{directories}{shared}{base}, directory => $anvil->data->{path}{directories}{shared}{base},
recursive => 1, recursive => 1,
}); });
@ -230,6 +239,7 @@ sub check_incoming
next if $file_type ne "file"; next if $file_type ne "file";
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name}; my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size}; my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime}; my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype}; my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
@ -287,13 +297,14 @@ sub check_incoming
# Insert or update the files entry. # Insert or update the files entry.
($file_uuid) = $anvil->Database->insert_or_update_files({ ($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2, debug => 2,
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_name => $file_name, file_name => $file_name,
file_size => $file_size, file_directory => $file_directory,
file_md5sum => $file_md5sum, file_size => $file_size,
file_mtime => $file_mtime, file_md5sum => $file_md5sum,
file_type => $say_mimetype, file_mtime => $file_mtime,
file_type => $say_mimetype,
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
} }
@ -334,6 +345,20 @@ sub check_incoming
source_file => $full_path, source_file => $full_path,
target_file => $target, target_file => $target,
}); });
# Update the file_directory.
my $say_directory =~ s/\/$//;
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $say_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
} }
} }
@ -476,6 +501,7 @@ sub handle_script
my $query = " my $query = "
SELECT SELECT
file_name, file_name,
file_directory,
file_size, file_size,
file_md5sum, file_md5sum,
file_type, file_type,
@ -489,17 +515,19 @@ WHERE
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
my $file_name = $results->[0]->[0]; my $file_name = $results->[0]->[0];
my $file_size = $results->[0]->[1]; my $file_directory = $results->[0]->[1];
my $file_md5sum = $results->[0]->[2]; my $file_size = $results->[0]->[2];
my $file_type = $results->[0]->[3]; my $file_md5sum = $results->[0]->[3];
my $file_mtime = $results->[0]->[4]; my $file_type = $results->[0]->[4];
my $file_mtime = $results->[0]->[5];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_name => $file_name, file_name => $file_name,
file_size => $file_size, file_directory => $file_directory,
file_md5sum => $file_md5sum, file_size => $file_size,
file_type => $file_type, file_md5sum => $file_md5sum,
file_mtime => $file_mtime, file_type => $file_type,
file_mtime => $file_mtime,
}}); }});
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'})) if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
{ {
@ -510,26 +538,28 @@ WHERE
my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0); my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
$anvil->Database->insert_or_update_files({ $anvil->Database->insert_or_update_files({
debug => 2, debug => 2,
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file}, file_name => $anvil->data->{switches}{file},
file_size => $file_size, file_directory => $file_directory,
file_md5sum => $file_md5sum, file_size => $file_size,
file_mtime => $file_mtime, file_md5sum => $file_md5sum,
file_type => $say_mimetype, file_mtime => $file_mtime,
file_type => $say_mimetype,
}); });
} }
elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'})) elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
{ {
# Change the file tpye to "script" # Change the file tpye to "script"
$anvil->Database->insert_or_update_files({ $anvil->Database->insert_or_update_files({
debug => 2, debug => 2,
file_uuid => $file_uuid, file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file}, file_name => $anvil->data->{switches}{file},
file_size => $file_size, file_directory => $file_directory,
file_md5sum => $file_md5sum, file_size => $file_size,
file_mtime => $file_mtime, file_md5sum => $file_md5sum,
file_type => "script", file_mtime => $file_mtime,
file_type => "script",
}); });
} }
} }

Write Preview
Loading…
Cancel
Save