From 1a36f37065261116787be823c56df2bd8429d816 Mon Sep 17 00:00:00 2001 From: Digimer Date: Fri, 1 Jan 2021 04:09:17 -0500 Subject: [PATCH] * Got file uploads working! * Got tools/striker-sync-shared to pick up 'upload::move_incoming' jobs, move the uploaded file to /mnt/shared/files/, copies it to peer dashboards, adds it to the 'files' table and adds it to 'file_locations'. * Reworked the 'file_locations' table to now map files to Anvil! systems, not hosts. It simply tracks if a given file should be on Anvil! members or not. Later, striker-sync-shared on the Anvil! members will pull the file down. * Updated Storage->get_file_stats() to record the file's mimetype. * Fixed up a few issues in cgi-bin/upload.pl. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + Anvil/Tools/Database.pm | 130 +++++++++--------- Anvil/Tools/Storage.pm | 9 +- cgi-bin/upload.pl | 60 +++++++-- share/anvil.sql | 59 +++------ share/words.xml | 12 ++ tools/anvil-provision-server | 6 +- tools/striker-sync-shared | 247 +++++++++++++++++++++++++++++++++++ 8 files changed, 398 insertions(+), 126 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index ae736b4f..7e839fdb 100644 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -1180,6 +1180,7 @@ sub _set_paths 'striker-parse-oui' => "/usr/sbin/striker-parse-oui", 'striker-prep-database' => "/usr/sbin/striker-prep-database", 'striker-scan-network' => "/usr/sbin/striker-scan-network", + 'striker-sync-shared' => "/usr/sbin/striker-sync-shared", stty => "/usr/bin/stty", su => "/usr/bin/su", 'subscription-manager' => "/usr/sbin/subscription-manager", diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index a51d5f25..b7bdc073 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -5763,7 +5763,7 @@ INSERT INTO This updates (or inserts) a record in the 'file_locations' table. The C<< file_location_uuid >> referencing the database row will be returned. -This table is used to track which of the files in the database are on a given host. +This table is used to track which files on Striker dashboards need to be on given Anvil! members. If there is an error, an empty string is returned. @@ -5771,21 +5771,21 @@ Parameters; =head3 file_location_uuid (optional) -The record to update, when passed. - -If not passed, a check will be made to see if an existing entry is found for C<< file_name >>. If found, that entry will be updated. If not found, a new record will be inserted. +If not passed, a check will be made to see if an existing entry is found for C<< file_location_file_uuid >>. If found, that entry will be updated. If not found, a new record will be inserted. =head3 file_location_file_uuid (required) -This is the C<< files >> -> C<< file_uuid >> referenced by this record. +This is the C<< files >> -> C<< file_uuid >> being referenced. -=head3 file_location_host_uuid (optional, default 'sys::host_uuid') +=head3 file_location_anvil_uuid (required) -This is the C<< hosts >> -> C<< host_uuid >> referenced by this record. +This is the C<< anvils >> -> C<< anvil_uuid >> being referenced. -=head3 uuid (optional) +=head3 file_location_active (required) -If set, only the corresponding database will be written to. +This is set to C<< 1 >> or C<< 0 >>, and indicates if the file should be on the Anvil! member machines or not. + +When set to C<< 1 >>, the file will be copied by the Anvil! member machines (by the member machines, they pull the files using rsync). If set to C<< 0 >>, the file is marked as inactive. If the file exists on the Anvil! members, it will be deleted. =cut sub insert_or_update_file_locations @@ -5796,17 +5796,21 @@ sub insert_or_update_file_locations my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->insert_or_update_file_locations()" }}); - my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : ""; - my $file = defined $parameter->{file} ? $parameter->{file} : ""; - my $line = defined $parameter->{line} ? $parameter->{line} : ""; - my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : ""; - my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : ""; - my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : $anvil->data->{sys}{host_uuid}; + my $uuid = defined $parameter->{uuid} ? $parameter->{uuid} : ""; + my $file = defined $parameter->{file} ? $parameter->{file} : ""; + my $line = defined $parameter->{line} ? $parameter->{line} : ""; + my $file_location_uuid = defined $parameter->{file_location_uuid} ? $parameter->{file_location_uuid} : ""; + my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : ""; + my $file_location_anvil_uuid = defined $parameter->{file_location_anvil_uuid} ? $parameter->{file_location_anvil_uuid} : ""; + my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - uuid => $uuid, - file_location_uuid => $file_location_uuid, - file_location_file_uuid => $file_location_file_uuid, - file_location_host_uuid => $file_location_host_uuid, + uuid => $uuid, + file => $file, + line => $line, + file_location_uuid => $file_location_uuid, + file_location_file_uuid => $file_location_file_uuid, + file_location_anvil_uuid => $file_location_anvil_uuid, + file_location_active => $file_location_active, }}); if (not $file_location_file_uuid) @@ -5815,15 +5819,18 @@ sub insert_or_update_file_locations $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_file_uuid" }}); return(""); } - if (not $file_location_host_uuid) + if (not $file_location_anvil_uuid) { # Throw an error and exit. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_host_uuid" }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_anvil_uuid" }}); + return(""); + } + if (($file_location_active ne "0") && ($file_location_active ne "1")) + { + # Throw an error and exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_active" }}); return(""); } - - # Made sure the file_uuuid and host_uuid are valid - ### TODO # If we don't have a UUID, see if we can find one for the given md5sum. if (not $file_location_uuid) @@ -5834,9 +5841,9 @@ SELECT FROM file_locations WHERE - file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid)." -AND - file_location_host_uuid = ".$anvil->Database->quote($file_location_host_uuid)." + file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid)." +AND + file_location_anvil_uuid = ".$anvil->Database->quote($file_location_anvil_uuid)." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); @@ -5857,28 +5864,6 @@ AND $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); if (not $file_location_uuid) { - # It's possible that this is called before the host is recorded in the database. So to be - # safe, we'll return without doing anything if there is no host_uuid in the database. - my $hosts = $anvil->Database->get_hosts({debug => $debug}); - my $found = 0; - foreach my $hash_ref (@{$hosts}) - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "hash_ref->{host_uuid}" => $hash_ref->{host_uuid}, - "sys::host_uuid" => $anvil->data->{sys}{host_uuid}, - }}); - if ($hash_ref->{host_uuid} eq $anvil->data->{sys}{host_uuid}) - { - $found = 1; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { found => $found }}); - } - } - if (not $found) - { - # We're out. - return(""); - } - # INSERT $file_location_uuid = $anvil->Get->uuid(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); @@ -5889,13 +5874,15 @@ INSERT INTO ( file_location_uuid, file_location_file_uuid, - file_location_host_uuid, + file_location_anvil_uuid, + file_location_active, modified_date ) VALUES ( ".$anvil->Database->quote($file_location_uuid).", ".$anvil->Database->quote($file_location_file_uuid).", - ".$anvil->Database->quote($file_location_host_uuid).", - ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." + ".$anvil->Database->quote($file_location_anvil_uuid).", + ".$anvil->Database->quote($file_location_active).", + ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." ); "; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); @@ -5907,7 +5894,8 @@ INSERT INTO my $query = " SELECT file_location_file_uuid, - file_location_host_uuid + file_location_anvil_uuid, + file_location_active, FROM file_locations WHERE @@ -5929,27 +5917,31 @@ WHERE } foreach my $row (@{$results}) { - my $old_file_location_file_uuid = $row->[0]; - my $old_file_location_host_uuid = $row->[1]; + my $old_file_location_file_uuid = $row->[0]; + my $old_file_location_anvil_uuid = $row->[1]; + my $old_file_location_active = $row->[2]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - old_file_location_file_uuid => $old_file_location_file_uuid, - old_file_location_host_uuid => $old_file_location_host_uuid, + old_file_location_file_uuid => $old_file_location_file_uuid, + old_file_location_anvil_uuid => $old_file_location_anvil_uuid, + old_file_location_active => $old_file_location_active, }}); # Anything change? - if (($old_file_location_file_uuid ne $file_location_file_uuid) or - ($old_file_location_host_uuid ne $file_location_host_uuid)) + if (($old_file_location_file_uuid ne $file_location_file_uuid) or + ($old_file_location_anvil_uuid ne $file_location_anvil_uuid) or + ($old_file_location_active ne $file_location_active)) { # Something changed, save. my $query = " UPDATE file_locations SET - file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid).", - file_location_host_uuid = ".$anvil->Database->quote($file_location_host_uuid).", - modified_date = ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." + file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid).", + file_location_anvil_uuid = ".$anvil->Database->quote($file_location_anvil_uuid).", + file_location_active = ".$anvil->Database->quote($file_location_active).", + modified_date = ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." WHERE - file_location_uuid = ".$anvil->Database->quote($file_location_uuid)." + file_location_uuid = ".$anvil->Database->quote($file_location_uuid)." "; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); $anvil->Database->write({uuid => $uuid, query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__}); @@ -6079,6 +6071,8 @@ SELECT FROM files WHERE + file_name = ".$anvil->Database->quote($file_name)." +AND file_md5sum = ".$anvil->Database->quote($file_md5sum)." ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); @@ -6100,6 +6094,7 @@ WHERE $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_uuid => $file_uuid }}); if (not $file_uuid) { +=cut Not sure why this is here... # It's possible that this is called before the host is recorded in the database. So to be # safe, we'll return without doing anything if there is no host_uuid in the database. my $hosts = $anvil->Database->get_hosts({debug => $debug}); @@ -6121,6 +6116,7 @@ WHERE # We're out. return(""); } +=cut # INSERT $file_uuid = $anvil->Get->uuid(); @@ -6186,10 +6182,10 @@ WHERE { my $old_file_name = $row->[0]; my $old_file_directory = $row->[1]; - my $old_file_size = $row->[1]; - my $old_file_md5sum = $row->[2]; - my $old_file_type = $row->[3]; - my $old_file_mtime = $row->[4]; + my $old_file_size = $row->[2]; + my $old_file_md5sum = $row->[3]; + my $old_file_type = $row->[4]; + my $old_file_mtime = $row->[5]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_file_name => $old_file_name, old_file_directory => $old_file_directory, diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 9c1c20aa..66bae034 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -1345,6 +1345,7 @@ Collected information is stored as (see C<< perldoc -f stat >> for details): file_stat::::inode_change_time file_stat::::block_size file_stat::::blocks + file_stat::::mimetype Parameters; @@ -1427,6 +1428,7 @@ sub get_file_stats $anvil->data->{file_stat}{$file_path}{inode_change_time} = $inode_change_time; $anvil->data->{file_stat}{$file_path}{block_size} = $block_size; $anvil->data->{file_stat}{$file_path}{blocks} = $blocks; + $anvil->data->{file_stat}{$file_path}{mimetype} = mimetype($file_path); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "s1:file_stat::${file_path}::device_number" => $anvil->data->{file_stat}{$file_path}{device_number}, @@ -1445,6 +1447,7 @@ sub get_file_stats "s14:file_stat::${file_path}::inode_change_time" => $anvil->data->{file_stat}{$file_path}{inode_change_time}, "s15:file_stat::${file_path}::block_size" => $anvil->data->{file_stat}{$file_path}{block_size}, "s16:file_stat::${file_path}::blocks" => $anvil->data->{file_stat}{$file_path}{blocks}, + "s17:file_stat::${file_path}::mimetype" => $anvil->data->{file_stat}{$file_path}{mimetype}, }}); return(0); @@ -2440,7 +2443,7 @@ Parameters; =head3 destination (required) -This is the source being copied. Be careful with the closing C<< / >>! Generally you will always want to have the destination end in a closing slash, to ensure the files go B<< under >> the estination directory. The same as is the case when using C<< rsync >> directly. +This is the destination being copied to. Be careful with the closing C<< / >>! Generally you will always want to have the destination end in a closing slash, to ensure the files go B<< under >> the estination directory. The same as is the case when using C<< rsync >> directly. =head3 password (optional) @@ -2452,9 +2455,11 @@ This is the TCP port used to connect to the target machine. =head3 source (required) +This is the file to copy via rsync. + The source can be a directory, or end in a wildcard (ie: C<< .../* >>) to copy multiple files/directories at the same time. -=head3 switches (optional, default -av) +=head3 switches (optional, default -avS) These are the switches to pass to C<< rsync >>. If you specify this and you still want C<< -avS >>, be sure to include it. This parameter replaces the default. diff --git a/cgi-bin/upload.pl b/cgi-bin/upload.pl index 65c7b0dd..0375e59a 100755 --- a/cgi-bin/upload.pl +++ b/cgi-bin/upload.pl @@ -30,6 +30,13 @@ my $cgi = CGI->new; print "Content-type: text/html; charset=utf-8\n\n"; print $anvil->Template->get({file => "files.html", name => "upload_header"})."\n"; +$anvil->Database->connect(); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + $anvil->nice_exit({exit_code => 1}); +} + my $lightweight_fh = $cgi->upload('field_name'); # undef may be returned if it's not a valid file handle if ($cgi->param()) @@ -37,19 +44,30 @@ if ($cgi->param()) my $start = time; my $filename = $cgi->upload('upload_file'); my $out_file = $anvil->data->{path}{directories}{shared}{incoming}."/".$filename; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { out_file => $out_file }}); if (-e $out_file) { # Don't overwrite $out_file .= "_".$anvil->Get->date_and_time({file_name => 1}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { out_file => $out_file }}); # If this exists (somehow), we'll append a short UUID if (-e $out_file) { $out_file .= "_".$anvil->Get->uuid({short => 1}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { out_file => $out_file }}); } } + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0259", variables => { file => $out_file }}); - my $cgi_file_handle = $cgi->param('upload_file'); + my $cgi_file_handle = $cgi->upload('upload_file'); + my $file = $cgi_file_handle; + my $mimetype = $cgi->uploadInfo($file)->{'Content-Type'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + cgi_file_handle => $cgi_file_handle, + file => $file, + mimetype => $mimetype, + }}); open(my $file_handle, ">$out_file") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0016", variables => { shell_call => $out_file, error => $! }}); while(<$cgi_file_handle>) { @@ -57,8 +75,23 @@ if ($cgi->param()) } close $file_handle; - # TODO: Call anvil-manage-files as a backgroup process, use the logic below and move the - $anvil->System->call({debug => 2, background => 1, shell_call => $anvil->data->{path}{exe}{'anvil-update-files'}}); + # Register a job to call striker-sync-shared + my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ + file => $THIS_FILE, + line => __LINE__, + job_command => $anvil->data->{path}{exe}{'striker-sync-shared'}, + job_data => "file=".$out_file, + job_name => "upload::move_incoming", + job_title => "job_0132", + job_description => "job_0133", + job_progress => 0, + job_host_uuid => $anvil->data->{sys}{host_uuid}, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }}); + + + + #$anvil->System->call({debug => 2, background => 1, shell_call => $anvil->data->{path}{exe}{'striker-sync-shared'}}); ### NOTE: The timing is a guide only. The AJAX does a lot of work before this script is invoked. It ### might be better to just remove the timing stuff entirely... @@ -71,7 +104,6 @@ if ($cgi->param()) my $bytes_per_second = $anvil->Convert->round({number => ($size / $took), places => 0}); my $say_rate = $anvil->Words->string({key => "suffix_0001", variables => { number => $anvil->Convert->bytes_to_human_readable({'bytes' => $bytes_per_second}) }}); my $file_sum = $anvil->Get->md5sum({file => $out_file}); - my $mimetype = mimetype($out_file); my $executable = -x $out_file ? 1 : 0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { size => $size, @@ -87,13 +119,13 @@ if ($cgi->param()) }}); # Determine the type (guess) from the mimetype - my $ype = "other"; + my $type = "other"; if ($mimetype =~ /cd-image/) { $type = "iso"; } # This will need to be expanded over time - elsif (($executable) or (($mimetype =~ /perl/) or ($mimetype =~ /python/)) + elsif (($executable) or ($mimetype =~ /perl/) or ($mimetype =~ /python/)) { $type = "script"; } @@ -118,13 +150,13 @@ if ($cgi->param()) if ($anvil->data->{sys}{database}{connections}) { # Add to files - my ($file_uuid) = $anvil->Database->insert_or_update_files({ - debug => 2, - file_name => $file_name, - file_size => $size, - file_md5sum => $file_sum, - file_type => $file_type, - }) +# my ($file_uuid) = $anvil->Database->insert_or_update_files({ +# debug => 2, +# file_name => $file_name, +# file_size => $size, +# file_md5sum => $file_sum, +# file_type => $file_type, +# }) } } else @@ -133,4 +165,4 @@ else $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0261", variables => { file => $THIS_FILE }}); } -exit(0); +$anvil->nice_exit({exit_code => 0}); diff --git a/share/anvil.sql b/share/anvil.sql index 3a655954..7001a00f 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -1106,7 +1106,7 @@ CREATE TABLE files ( file_directory text not null, -- This is the directory that the file is in. file_size numeric not null, -- This is the file's size in bytes. If it recorded as a quick way to determine if a file has changed on disk. file_md5sum text not null, -- This is the sum as calculated when the file is first uploaded. Once recorded, it can't change. - file_type text not null, -- This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 'other'. + file_type text not null, -- This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or 'other'. If set to 'DELETED', the file will be removed from disk. file_mtime numeric not null, -- If the same file exists on different machines and differ md5sums/sizes, the one with the most recent mtime will be used to update the others. modified_date timestamp with time zone not null ); @@ -1160,51 +1160,26 @@ CREATE TRIGGER trigger_files FOR EACH ROW EXECUTE PROCEDURE history_files(); --- NOTE: When an entry is made here, the next time files are checked on a machine and an entry doesn't exist --- on disk, the file fill be found (if possible) and copied to the houst. Only machines on the same --- subnet are searched. Of course, if a URL is given (or a file is uploaded over a browser), the file --- will be sourced accordingly. The search pattern is; --- Nodes; 1. Check for the file on the peer. --- 2. Check for the file on Strikers, in alphabetical order. --- 3. Check for the file on DR host, if available. --- 4. Check other nodes, in alphabetical order. --- 5. Check other DR hosts, in alphabetical order. --- Striker; 1. Check for the file on other Strikers, in alphabetical order. --- 2. Check for the file on DR hosts, if available --- 3. Check for the file on Anvil! nodes. --- DR Host; 1. Check for the file on Strikers, in alphabetical order. --- 2. Check for the file on Anvil! nodes. --- * If a file can't be found, it will try again every so often until it is found. --- * When a file is found, it is copied to '/mnt/shared/incoming'. Only when the file has arrived and --- the md5sum matches. At this point, it is moved into the proper directory. --- How new files are handled; --- * When uploading a file from a Striker web interface, or when creating an ISO from physical media, --- it will be dropped into /mnt/shared/incoming. Once there, the user will have the option of pushing --- the file to an Anvil! system. ISOs and scripts will go to both nodes (and the DR host, when --- needed). --- * Repo RPMs are sync'ed to all peer'ed dashboards, but not sent to hosts (they are used during the --- initial host setup). --- * Special Note: Definition files are stored in the database and written out as needed to the --- nodes/DR host. --- --- This tracks which files should be on which machines. +-- This tracks which files on Strikers should be on Anvils! CREATE TABLE file_locations ( - file_location_uuid uuid not null primary key, - file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine. - file_location_host_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change. - modified_date timestamp with time zone not null, + file_location_uuid uuid not null primary key, + file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine. + file_location_anvil_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change. + file_location_active boolean not null default TRUE, -- This is set to true when the file should be on Anvil! machines, triggering rsyncs when needed. When set to false, the file will be deleted from members, if they exist. + modified_date timestamp with time zone not null, FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid), - FOREIGN KEY(file_location_host_uuid) REFERENCES hosts(host_uuid) + FOREIGN KEY(file_location_anvil_uuid) REFERENCES anvils(anvil_uuid) ); ALTER TABLE file_locations OWNER TO admin; CREATE TABLE history.file_locations ( - history_id bigserial, - file_location_uuid uuid, - file_location_file_uuid text, - file_location_host_uuid text, - modified_date timestamp with time zone not null + history_id bigserial, + file_location_uuid uuid, + file_location_file_uuid text, + file_location_anvil_uuid uuid, + file_location_active boolean, + modified_date timestamp with time zone not null ); ALTER TABLE history.file_locations OWNER TO admin; @@ -1217,12 +1192,14 @@ BEGIN INSERT INTO history.file_locations (file_location_uuid, file_location_file_uuid, - file_location_host_uuid, + file_location_anvil_uuid, + file_location_active, modified_date) VALUES (history_file_locations.file_location_uuid, history_file_locations.file_location_file_uuid, - history_file_locations.file_location_host_uuid, + history_file_locations.file_location_anvil_uuid, + history_file_locations.file_location_active, history_file_locations.modified_date); RETURN NULL; END; diff --git a/share/words.xml b/share/words.xml index 2678899c..b7d8c3cf 100644 --- a/share/words.xml +++ b/share/words.xml @@ -242,6 +242,9 @@ The error was: The resource: [#!variable!resource!#] in the config file: [#!variable!file!#] was found, and we were asked to replace the 'scan_drbd_resource_uuid' but the new UUID: [#!variable!uuid!#] is not a valud UUID. The 'fence_ipmilan' command: [#!variable!command!#] does not appear to be valid. The Anvil! UUID: [#!variable!anvil_uuid!#] doesn't appear to exist in the database. + Unable to move an uploaded file from the: [#!data!path::directories::shared::incoming!#] directory as a file name wasn't set (or failed to parse) from the 'job_data' in the job: [#!variable!job_uuid!#]. + Unable to move the uploaded file: [#!variable!file!#], it doesn't appear to exist. + Unable to move the uploaded file: [#!variable!file!#] to: [#!variable!target_directory!#]. The cause of the failure should be in the logs. Current Network Interfaces and States @@ -422,6 +425,8 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec Completed joining the #!string!brand_0002!#. No job was found that needs to be run. Reconnecting will start a synchonization of the database. This step might take a while to complete, please be patient. + Sync Uploaded File + This moves an uploaded file from the: [#!data!path::directories::shared::incoming!#] directory to: [#!data!path::directories::shared::files!#] directory, adds it to the Anvil! database, and pushed it out to other systems. Starting: [#!variable!program!#]. @@ -1376,6 +1381,13 @@ About to try to download aproximately: [#!variable!packages!#] packages needed t # scan_drbd_resource_uuid = #!variable!uuid!# Preparing to provision a new server. + Processing an uploaded file. + Moving the file: [#!variable!file!#] to: [#!data!path::directories::shared::files!#]. + Calculating the md5sum. This can take a little while for large files, please be patient. + The md5sum is: [#!variable!md5sum!#]. Storing details in the database. + Copying the file over to: [#!variable!host!#]. Please be patient, this could take a bit for large files. + Registering the file to be downloaded to the Anvil!: [#!variable!anvil_name!#]. Anvil! members will sync this file shortly. Member machines that are not online will sync the file when they do return. + Upload is complete! Saved the mail server information successfully! diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 0b40438f..5bd07463 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -74,8 +74,10 @@ if ($anvil->data->{switches}{'job-uuid'}) $anvil->Job->clear(); $anvil->Job->get_job_details(); $anvil->Job->update_progress({ - progress => 1, - message => "message_0190", + progress => 1, + job_picked_up_by => $$, + job_picked_up_at => time, + message => "message_0190", }); # Job data will be in $anvil->data->{jobs}{job_data} diff --git a/tools/striker-sync-shared b/tools/striker-sync-shared index ffbfe2f2..14625160 100755 --- a/tools/striker-sync-shared +++ b/tools/striker-sync-shared @@ -4,6 +4,9 @@ # pulls over any files under /mnt/shared/files/ to the same on the local system. It then pushes files out to # all members of the same Anvil!. # +# If this is called with a job-uuid, file-specific tasks will be handled, like moving files uploaded over a +# browser or deleting / purging a file. +# # NOTE: This file is NOT responsible for sync'ing definition files! That is handles in scan-server. # # TODO: @@ -30,12 +33,256 @@ my $anvil = Anvil::Tools->new(); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); # Read switches (target ([user@]host[:port]) and the file with the target's password. +$anvil->data->{switches}{'job-uuid'} = ""; $anvil->Get->switches; # Connect to the database(s). $anvil->Database->connect; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"}); +# If we don't have a job-uuid, look for one. +if (not $anvil->data->{switches}{'job-uuid'}) +{ + # Load the job data. + $anvil->data->{switches}{'job-uuid'} = $anvil->Job->get_job_uuid({debug => 2, program => $THIS_FILE}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }}); +} + +# If we still don't have a job-uuit, go into interactive mode. +if ($anvil->data->{switches}{'job-uuid'}) +{ + # Load the job data. + $anvil->Job->get_job_details({debug => 2}); + $anvil->Job->clear(); + + $anvil->data->{sys}{progress} = 1; + if ($anvil->data->{jobs}{job_name} eq "upload::move_incoming") + { + process_incoming_file($anvil); + } + + # Job data will be in $anvil->data->{jobs}{job_data} +} +else +{ + # Do a normal periodic search. + ### NOTE: When finding new files, check the size, sleep for 30 seconds, and check again. If a file's + ### size changed, skip it, it's likely still being updated. +} $anvil->nice_exit({exit_code => 0}); + +############################################################################################################# +# Functions # +############################################################################################################# + +sub process_incoming_file +{ + my ($anvil) = @_; + + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0191", + }); + + my $file = ($anvil->data->{jobs}{job_data} =~ /file=(.*)$/)[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file => $file }}); + if (not $file) + { + # Can't do anything, file wasn't parsed. + $anvil->Job->update_progress({ + progress => 100, + message => "error_0170,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", + job_status => "failed", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0170", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }}); + + $anvil->nice_exit({exit_code => 1}); + } + elsif (not -e $file) + { + # Can't do anything, file doesn't exist + $anvil->Job->update_progress({ + progress => 100, + message => "error_0171,!!file!".$file."!!", + job_status => "failed", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }}); + $anvil->nice_exit({exit_code => 1}); + } + + # Move it over to files. + $anvil->data->{sys}{progress} = 10; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0192,!!file!".$file."!!", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0192", variables => { file => $file }}); + + $anvil->Storage->move_file({ + debug => 2, + overwrite => 1, + source_file => $file, + target_file => $anvil->data->{path}{directories}{shared}{files}."/", + }); + + my $file_name = ($file =~ /\/.*\/(.*?)$/)[0]; + my $target_file = $anvil->data->{path}{directories}{shared}{files}."/".$file_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + file_name => $file_name, + target_file => $target_file, + }}); + + if (not -e $target_file) + { + # Failed to move. + $anvil->Job->update_progress({ + progress => 100, + message => "error_0172,!!file!".$file."!!,!!target_directory!".$anvil->data->{path}{directories}{shared}{files}."!!", + job_status => "failed", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0171", variables => { file => $file }}); + $anvil->nice_exit({exit_code => 1}); + } + + # Calculate the md5sum. + $anvil->data->{sys}{progress} = 20; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0193", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0193"}); + + my ($string, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{md5sum}." ".$target_file}); + my $md5sum = ($string =~ /^(.*?)\s/)[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + string => $string, + md5sum => $md5sum, + return_code => $return_code, + }}); + + # Store the file details! + $anvil->data->{sys}{progress} = 30; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0194,!!md5sum!".$md5sum."!!", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0194", variables => { md5sum => $md5sum }}); + + $anvil->Storage->get_file_stats({ + debug => 2, + file_path => $target_file, + }); + + my $file_mimetype = $anvil->data->{file_stat}{$target_file}{mimetype}; + my $file_size = $anvil->data->{file_stat}{$target_file}{size}; + my $file_mtime = $anvil->data->{file_stat}{$target_file}{modified_time}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + file_mimetype => $file_mimetype, + file_size => $file_size." (".$anvil->Convert->bytes_to_human_readable({"bytes" => $file_size}).")", + file_mtime => $file_mtime, + }}); + + # This is the file's type/purpose. The expected values are 'iso', 'rpm', 'script', 'disk-image', or + # 'other'. If set to 'DELETED', the file will be removed from disk. + my $file_type = "other"; + if ($file_mimetype =~ /cd-image/) + { + $file_type = "iso"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_type => $file_type }}); + } + + my $file_uuid = $anvil->Database->insert_or_update_files({ + debug => 2, + file_name => $file_name, + file_directory => $anvil->data->{path}{directories}{shared}{files}, + file_size => $file_size, + file_md5sum => $md5sum, + file_type => $file_type, + file_mtime => $file_mtime, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); + + # Now copy this to our peers. + foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}}) + { + # Periodically, autovivication causes and empty key to appear. + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid => $uuid }}); + next if ((not $uuid) or (not $anvil->Validate->uuid({uuid => $uuid}))); + next if $uuid eq $anvil->Get->host_uuid; + + my $host = $anvil->data->{database}{$uuid}{host}; + my $password = $anvil->data->{database}{$uuid}{password}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + host => $host, + password => $anvil->Log->is_secure($password), + }}); + + my $striker_name = $anvil->Get->host_name_from_uuid({host_uuid => $uuid}); + my $say_host = $striker_name." (".$host.")"; + + # Rsync the file. + $anvil->data->{sys}{progress} += 10; + $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0195,!!host!".$say_host."!!", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0195", variables => { host => $say_host }}); + + $anvil->Storage->rsync({ + debug => 2, + source => $target_file, + destination => "root\@".$host.":".$anvil->data->{path}{directories}{shared}{files}."/", + try_again => 1, + }); + } + + ### TODO: Make is an upload-time option to choose if the uploaded file automatically goes to any given Anvil! + # Tell other Anvil! systems to download this file. + $anvil->Database->get_anvils({debug => 2}); + foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}}) + { + my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid}; + my $anvil_node1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node1_host_uuid}; + my $anvil_node2_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_node2_host_uuid}; + my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + anvil_uuid => $anvil_uuid, + anvil_node1_host_uuid => $anvil_node1_host_uuid, + anvil_node2_host_uuid => $anvil_node2_host_uuid, + anvil_dr1_host_uuid => $anvil_dr1_host_uuid, + }}); + + $anvil->data->{sys}{progress} += 5; + $anvil->data->{sys}{progress} = 90 if $anvil->data->{sys}{progress} > 90; + $anvil->Job->update_progress({ + progress => $anvil->data->{sys}{progress}, + message => "message_0196,!!anvil_name!".$anvil_name."!!", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0196", variables => { anvil_name => $anvil_name }}); + + my $file_location_uuid = $anvil->Database->insert_or_update_file_locations({ + debug => 2, + file_location_file_uuid => $file_uuid, + file_location_anvil_uuid => $anvil_uuid, + file_location_active => 1, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); + + ### TODO: Register a job for each member to run this file with an 'upload::pull_file' with + ### the job_data being the 'file=$target_file'. Have node1 try the first striker, and + ### node2 from the second striker, if it exists. DR should watch node1 and node2's + ### jobs and not download until those jobs hit 100. + } + + # Done! + $anvil->Job->update_progress({ + progress => 100, + message => "message_0197", + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "message_0197"}); + + return(0); +}