seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Added 'files -> file_directory' to help distinquish cases where two different files of the same name exist in different directories.

Browse Source 
Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent 529c12d2e2
commit 47280c8fd9
3 changed files with 153 additions and 96 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 95
      Anvil/Tools/Database.pm
  2. 4
      share/anvil.sql
  3. 150
      tools/anvil-manage-files

95
Anvil/Tools/Database.pm
Unescape View File

@ -2389,6 +2389,10 @@ If not passed, a check will be made to see if an existing entry is found for C<<
This is the file's name.
=head3 file_directory (required)
This is the directory that the file is in. This is used to avoid conflict if two files of the same name exist in two places but are otherwise different.
=head3 file_size (required)
This is the file's size in bytes. It is recorded as a quick way to determine if the file has changed on disk.
@ -2414,21 +2418,23 @@ sub insert_or_update_files
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_files()" }});
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : "";
my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : "";
my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : "";
my $file_md5sum = defined $parameter->{file_md5sum} ? $parameter->{file_md5sum} : "";
my $file_type = defined $parameter->{file_type} ? $parameter->{file_type} : "";
my $file_mtime = defined $parameter->{file_mtime} ? $parameter->{file_mtime} : "";
my $file = defined $parameter->{file} ? $parameter->{file} : "";
my $line = defined $parameter->{line} ? $parameter->{line} : "";
my $file_uuid = defined $parameter->{file_uuid} ? $parameter->{file_uuid} : "";
my $file_name = defined $parameter->{file_name} ? $parameter->{file_name} : "";
my $file_directory = defined $parameter->{file_directory} ? $parameter->{file_directory} : "";
my $file_size = defined $parameter->{file_size} ? $parameter->{file_size} : "";
my $file_md5sum = defined $parameter->{file_md5sum} ? $parameter->{file_md5sum} : "";
my $file_type = defined $parameter->{file_type} ? $parameter->{file_type} : "";
my $file_mtime = defined $parameter->{file_mtime} ? $parameter->{file_mtime} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_uuid => $file_uuid,
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
}});
if (not $file_name)
@ -2443,6 +2449,12 @@ sub insert_or_update_files
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_name" }});
return("");
}
if (not $file_directory)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_files()", parameter => "file_directory" }});
return("");
}
if (not $file_md5sum)
{
# Throw an error and exit.
@ -2524,6 +2536,7 @@ INSERT INTO
(
file_uuid,
file_name,
file_directory,
file_size,
file_md5sum,
file_type,
@ -2532,6 +2545,7 @@ INSERT INTO
) VALUES (
".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_name).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_directory).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_size).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
@ -2548,6 +2562,7 @@ INSERT INTO
my $query = "
SELECT
file_name,
file_directory,
file_size,
file_md5sum,
file_type,
@ -2573,39 +2588,43 @@ WHERE
}
foreach my $row (@{$results})
{
my $old_file_name = $row->[0];
my $old_file_size = $row->[1];
my $old_file_md5sum = $row->[2];
my $old_file_type = $row->[3];
my $old_file_mtime = $row->[4];
my $old_file_name = $row->[0];
my $old_file_directory = $row->[1];
my $old_file_size = $row->[1];
my $old_file_md5sum = $row->[2];
my $old_file_type = $row->[3];
my $old_file_mtime = $row->[4];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
old_file_name => $old_file_name,
old_file_size => $old_file_size,
old_file_md5sum => $old_file_md5sum,
old_file_type => $old_file_type,
old_file_mtime => $old_file_mtime,
old_file_name => $old_file_name,
old_file_directory => $old_file_directory,
old_file_size => $old_file_size,
old_file_md5sum => $old_file_md5sum,
old_file_type => $old_file_type,
old_file_mtime => $old_file_mtime,
}});
# Anything change?
if (($old_file_name ne $file_name) or
($old_file_size ne $file_size) or
($old_file_md5sum ne $file_md5sum) or
($old_file_mtime ne $file_mtime) or
($old_file_type ne $file_type))
if (($old_file_name ne $file_name) or
($old_file_directory ne $file_directory) or
($old_file_size ne $file_size) or
($old_file_md5sum ne $file_md5sum) or
($old_file_mtime ne $file_mtime) or
($old_file_type ne $file_type))
{
# Something changed, save.
my $query = "
UPDATE
files
SET
file_name = ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).",
file_size = ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).",
file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
file_type = ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
file_mtime = ".$anvil->data->{sys}{database}{use_handle}->quote($file_mtime).",
modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
file_name = ".$anvil->data->{sys}{database}{use_handle}->quote($file_name).",
file_directory = ".$anvil->data->{sys}{database}{use_handle}->quote($file_directory).",
file_size = ".$anvil->data->{sys}{database}{use_handle}->quote($file_size).",
file_md5sum = ".$anvil->data->{sys}{database}{use_handle}->quote($file_md5sum).",
file_type = ".$anvil->data->{sys}{database}{use_handle}->quote($file_type).",
file_mtime = ".$anvil->data->{sys}{database}{use_handle}->quote($file_mtime).",
modified_date = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{database}{timestamp})."
WHERE
file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)."
file_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($file_uuid)."
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__});
@ -5347,6 +5366,9 @@ sub query
$anvil->Database->check_lock_age({debug => $debug});
# Do I need to log the transaction?
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::log_transactions" => $anvil->data->{sys}{database}{log_transactions},
}});
if ($anvil->data->{sys}{database}{log_transactions})
{
$anvil->Log->entry({source => $source, line => $line, secure => $secure, level => 0, key => "log_0074", variables => {
@ -5361,6 +5383,7 @@ sub query
server => $say_server,
db_error => $DBI::errstr,
}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { DBreq => $DBreq }});
# Execute on the query
$DBreq->execute() or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0076", variables => {

4
share/anvil.sql
Unescape View File

@ -1102,6 +1102,7 @@ CREATE TRIGGER trigger_ip_addresses
CREATE TABLE files (
file_uuid uuid not null primary key,
file_name text not null, -- This is the file's name. It can change without re-uploading the file.
file_directory text not null, -- This is the directory that the file is in.
file_size numeric not null, -- This is the file's size in bytes. If it recorded as a quick way to determine if a file has changed on disk.
file_md5sum text not null, -- This is the sum as calculated when the file is first uploaded. Once recorded, it can't change.
file_type text not null, -- This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 'other'.
@ -1114,6 +1115,7 @@ CREATE TABLE history.files (
history_id bigserial,
file_uuid uuid,
file_name text,
file_directory text,
file_size numeric,
file_md5sum text,
file_type text,
@ -1131,6 +1133,7 @@ BEGIN
INSERT INTO history.files
(file_uuid,
file_name,
file_directory,
file_size,
file_md5sum,
file_type,
@ -1139,6 +1142,7 @@ BEGIN
VALUES
(history_files.file_uuid,
history_files.file_name,
history_files.file_directory,
history_files.file_size,
history_files.file_md5sum,
history_files.file_type,

150
tools/anvil-manage-files
Unescape View File

@ -40,7 +40,6 @@ use warnings;
use Anvil::Tools;
use Data::Dumper;
# Disable buffering
$| = 1;
@ -167,33 +166,43 @@ sub find_missing_files
my ($anvil) = @_;
# What am I? This will impact how missing files are found.
# my $query = "
# SELECT
# file_location_file_uuid
# FROM
# file_locations
# WHERE
# file_location_host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})."
# ;";
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
#
# my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
# my $count = @{$results};
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
# results => $results,
# count => $count,
# }});
# foreach my $row (@{$results})
# {
# my $file_location_file_uuid = $row->[0];
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
# file_location_file_uuid => $file_location_file_uuid,
# }});
#
# ### TODO: How to handle when the file with the same name exists on 2+ machines with
# ### different md5sums. Use the most recent mtime?
# # Read in the file details.
# }
my $query = "
SELECT
a.file_uuid,
a.file_name
FROM
files a,
hosts b,
file_locations c
WHERE
b.host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($anvil->data->{sys}{host_uuid})."
AND
a.file_uuid = c.file_location_file_uuid
AND
b.host_uuid = c.file_location_host_uuid
ORDER BY
a.file_name ASC,
b.host_name ASC
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $file_location_file_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_location_file_uuid => $file_location_file_uuid,
}});
### TODO: How to handle when the file with the same name exists on 2+ machines with
### different md5sums. Use the most recent mtime?
# Read in the file details.
}
# Read in any entries from 'file_locations'.
@ -212,7 +221,7 @@ sub check_incoming
}
# Read any files in '/mnt/shared/incoming'.
$anvil->Storage->scan_directory({
debug => 2,
debug => 1,
directory => $anvil->data->{path}{directories}{shared}{base},
recursive => 1,
});
@ -230,6 +239,7 @@ sub check_incoming
next if $file_type ne "file";
my $file_name = $anvil->data->{scan}{directories}{$full_path}{name};
my $file_directory = $anvil->data->{scan}{directories}{$full_path}{directory};
my $file_size = $anvil->data->{scan}{directories}{$full_path}{size};
my $file_mtime = $anvil->data->{scan}{directories}{$full_path}{mtime};
my $file_mimetype = $anvil->data->{scan}{directories}{$full_path}{mimetype};
@ -287,13 +297,14 @@ sub check_incoming
# Insert or update the files entry.
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
debug => 2,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
}
@ -334,6 +345,20 @@ sub check_incoming
source_file => $full_path,
target_file => $target,
});
# Update the file_directory.
my $say_directory =~ s/\/$//;
($file_uuid) = $anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $file_name,
file_directory => $say_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
}
}
@ -476,6 +501,7 @@ sub handle_script
my $query = "
SELECT
file_name,
file_directory,
file_size,
file_md5sum,
file_type,
@ -489,17 +515,19 @@ WHERE
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { results => $results }});
my $file_name = $results->[0]->[0];
my $file_size = $results->[0]->[1];
my $file_md5sum = $results->[0]->[2];
my $file_type = $results->[0]->[3];
my $file_mtime = $results->[0]->[4];
my $file_name = $results->[0]->[0];
my $file_directory = $results->[0]->[1];
my $file_size = $results->[0]->[2];
my $file_md5sum = $results->[0]->[3];
my $file_type = $results->[0]->[4];
my $file_mtime = $results->[0]->[5];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_name => $file_name,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
file_name => $file_name,
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_type => $file_type,
file_mtime => $file_mtime,
}});
if (($file_type eq "script") && (not $anvil->data->{switches}{'is-script'}))
{
@ -510,26 +538,28 @@ WHERE
my $say_mimetype = convert_mimetype($anvil, $mimetype, $file_name, 0);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_mimetype => $say_mimetype }});
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => $say_mimetype,
});
}
elsif (($file_type ne "script") && ($anvil->data->{switches}{'is-script'}))
{
# Change the file tpye to "script"
$anvil->Database->insert_or_update_files({
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => "script",
debug => 2,
file_uuid => $file_uuid,
file_name => $anvil->data->{switches}{file},
file_directory => $file_directory,
file_size => $file_size,
file_md5sum => $file_md5sum,
file_mtime => $file_mtime,
file_type => "script",
});
}
}

Write Preview
Loading…
Cancel
Save