From c50a1936c0aa590d6dbc2cd967f0234ebf8a4ee3 Mon Sep 17 00:00:00 2001 From: digimer Date: Thu, 4 May 2023 00:05:56 -0400 Subject: [PATCH 1/3] * This adds the new 'file_locations' -> 'file_location_ready' column and associated methods. This is set to TRUE/1 when the file referenced is found on disk and it is the expected size and md5sum. This is meant to allow programs to wait/watch or a file to be ready if they need to use it. Files are now checked periodically via anvil-daemon. * Removed hard-coded log levels in anvil-provision-server and anvil-manage-storage-groups. Signed-off-by: digimer --- Anvil/Tools/Database.pm | 55 +++++++++++- Anvil/Tools/Storage.pm | 138 ++++++++++++++++++++++++++++++ notes | 2 +- share/anvil.sql | 4 + tools/anvil-daemon | 81 +++++++++++------- tools/anvil-manage-storage-groups | 2 - tools/anvil-provision-server | 2 - tools/anvil-version-changes | 102 +++++++++++++++++++++- 8 files changed, 342 insertions(+), 44 deletions(-) diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 8d0dba12..5c22aebe 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -429,6 +429,7 @@ sub check_file_locations file_location_file_uuid => $file_uuid, file_location_host_uuid => $host_uuid, file_location_active => 1, + file_location_ready => "same", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); } @@ -3487,6 +3488,7 @@ This loads the known install file_locations into the C<< anvil::data >> hash at: * file_locations::file_location_uuid::::file_location_file_uuid * file_locations::file_location_uuid::::file_location_host_uuid * file_locations::file_location_uuid::::file_location_active +* file_locations::file_location_uuid::::file_location_ready * file_locations::file_location_uuid::::modified_date If the hash was already populated, it is cleared before repopulating to ensure no stale data remains. @@ -3513,6 +3515,7 @@ SELECT file_location_file_uuid, file_location_host_uuid, file_location_active, + file_location_ready, modified_date FROM file_locations @@ -3530,12 +3533,14 @@ FROM my $file_location_file_uuid = $row->[1]; my $file_location_host_uuid = $row->[2]; my $file_location_active = $row->[3]; - my $modified_date = $row->[4]; + my $file_location_ready = $row->[4]; + my $modified_date = $row->[5]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid, file_location_file_uuid => $file_location_file_uuid, file_location_host_uuid => $file_location_host_uuid, file_location_active => $file_location_active, + file_location_ready => $file_location_ready, modified_date => $modified_date, }}); @@ -3543,11 +3548,13 @@ FROM $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid} = $file_location_file_uuid; $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid} = $file_location_host_uuid; $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active} = $file_location_active; + $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready} = $file_location_ready; $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date} = $modified_date; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "file_locations::file_location_uuid::${file_location_uuid}::file_location_file_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid}, "file_locations::file_location_uuid::${file_location_uuid}::file_location_host_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid}, "file_locations::file_location_uuid::${file_location_uuid}::file_location_active" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}, + "file_locations::file_location_uuid::${file_location_uuid}::file_location_ready" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready}, "file_locations::file_location_uuid::${file_location_uuid}::modified_date" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date}, }}); @@ -8490,6 +8497,14 @@ This is set to C<< 1 >> or C<< 0 >>, and indicates if the file should be on the When set to C<< 1 >>, the file will be copied by the Anvil! member machines (by the member machines, they pull the files using rsync). If set to C<< 0 >>, the file is marked as inactive. If the file exists on the Anvil! members, it will be deleted. +=head3 file_location_ready (optional, default '0') + +This is set to C<< 1 >> or C<< 0 >>, and indicates if the file is on the system and ready to be used. + +B<< Note >>: This can also be set to C<< same >>. If set, and the file exists in the database, the existing value is retained. If the entry is inserted, this is set to C<< 0 >>. + +When set to C<< 1 >>, the file's size and md5sum have been confirmed to match on disk what is recorded in the database. When set to C<< 0 >>, the file _may_ be ready, but it probably isn't yet. Any process needing the file should check that it's ready before using it. + =cut sub insert_or_update_file_locations { @@ -8506,7 +8521,8 @@ sub insert_or_update_file_locations my $file_location_anvil_uuid = defined $parameter->{file_location_anvil_uuid} ? $parameter->{file_location_anvil_uuid} : ""; my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : ""; my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : ""; - my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : ""; + my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : 0; + my $file_location_ready = defined $parameter->{file_location_ready} ? $parameter->{file_location_ready} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid, file => $file, @@ -8516,6 +8532,7 @@ sub insert_or_update_file_locations file_location_file_uuid => $file_location_file_uuid, file_location_host_uuid => $file_location_host_uuid, file_location_active => $file_location_active, + file_location_ready => $file_location_ready, }}); if (not $file_location_file_uuid) @@ -8536,6 +8553,12 @@ sub insert_or_update_file_locations $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_active" }}); return(""); } + if (($file_location_ready ne "0") && ($file_location_ready ne "1") && ($file_location_ready ne "same")) + { + # Throw an error and exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_ready" }}); + return(""); + } # If we've got an Anvil! uuid, find out the hosts and DR links connected to the Anvil! are found and # this method is recursively called for each host. @@ -8577,6 +8600,7 @@ sub insert_or_update_file_locations file_location_file_uuid => $file_location_file_uuid, file_location_host_uuid => $host_uuid, file_location_active => $file_location_active, + file_location_ready => $file_location_ready, }); $file_location_uuids .= $host_uuid."=".$file_location_uuid.","; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { @@ -8626,6 +8650,12 @@ AND $file_location_uuid = $anvil->Get->uuid(); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); + if ($file_location_ready eq "same") + { + $file_location_ready = 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_ready => $file_location_ready }}); + } + my $query = " INSERT INTO file_locations @@ -8634,12 +8664,14 @@ INSERT INTO file_location_file_uuid, file_location_host_uuid, file_location_active, + file_location_ready, modified_date ) VALUES ( ".$anvil->Database->quote($file_location_uuid).", ".$anvil->Database->quote($file_location_file_uuid).", ".$anvil->Database->quote($file_location_host_uuid).", ".$anvil->Database->quote($file_location_active).", + ".$anvil->Database->quote($file_location_ready).", ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." ); "; @@ -8653,7 +8685,8 @@ INSERT INTO SELECT file_location_file_uuid, file_location_host_uuid, - file_location_active + file_location_active, + file_location_ready FROM file_locations WHERE @@ -8678,16 +8711,25 @@ WHERE my $old_file_location_file_uuid = $row->[0]; my $old_file_location_host_uuid = $row->[1]; my $old_file_location_active = $row->[2]; + my $old_file_location_ready = $row->[3]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_file_location_file_uuid => $old_file_location_file_uuid, old_file_location_host_uuid => $old_file_location_host_uuid, old_file_location_active => $old_file_location_active, + old_file_location_ready => $old_file_location_ready, }}); + if ($file_location_ready eq "same") + { + $file_location_ready = $old_file_location_ready; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_ready => $file_location_ready }}); + } + # Anything change? if (($old_file_location_file_uuid ne $file_location_file_uuid) or ($old_file_location_host_uuid ne $file_location_host_uuid) or - ($old_file_location_active ne $file_location_active)) + ($old_file_location_active ne $file_location_active) or + ($old_file_location_ready ne $file_location_ready)) { # Something changed, save. my $query = " @@ -8697,6 +8739,7 @@ SET file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid).", file_location_host_uuid = ".$anvil->Database->quote($file_location_host_uuid).", file_location_active = ".$anvil->Database->quote($file_location_active).", + file_location_ready = ".$anvil->Database->quote($file_location_ready).", modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." WHERE file_location_uuid = ".$anvil->Database->quote($file_location_uuid)." @@ -18185,6 +18228,7 @@ sub track_files file_location_file_uuid => $file_uuid, file_location_host_uuid => $host_uuid, file_location_active => 1, + file_location_ready => "same", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { reload => $reload, @@ -18250,6 +18294,7 @@ sub track_files file_location_file_uuid => $file_uuid, file_location_host_uuid => $host_uuid, file_location_active => 1, + file_location_ready => "same", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); } @@ -18267,6 +18312,7 @@ sub track_files file_location_file_uuid => $file_uuid, file_location_host_uuid => $host_uuid, file_location_active => 1, + file_location_ready => "same", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); } @@ -18323,6 +18369,7 @@ sub track_files file_location_file_uuid => $file_uuid, file_location_host_uuid => $host_uuid, file_location_active => 1, + file_location_ready => "same", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); } diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 92595c26..1588a847 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -19,6 +19,7 @@ my $THIS_FILE = "Storage.pm"; # backup # change_mode # change_owner +# check_files # check_md5sums # compress # copy_file @@ -583,6 +584,142 @@ sub change_owner return($error); } + +=head2 check_files + +This method checks the files on the local system. Specifically, it looks in C<< file_locations >> table and then checks if the file is "ready" or not. Depending on the results, C<< file_location_ready >> is updated if needed. + +This method takes no parameters. + +=cut +sub check_files +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->check_md5sums()" }}); + + $anvil->Database->get_files({debug => $debug}); + $anvil->Database->get_file_locations({debug => $debug}); + + # Look for files that should be on this host. + my $host_uuid = $anvil->Get->host_uuid({debug => $debug}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }}); + + # Sorting isn't useful really, but it ensures consistent listing run over run). + foreach my $file_location_file_uuid (sort {$a cmp $b} keys %{$anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}}) + { + my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid}; + my $file_location_file_uuid = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid}; + my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}; + my $file_location_ready = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready}; + my $file_name = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_name}; + my $file_directory = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_directory}; + my $file_size = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_size}; + my $file_md5sum = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_md5sum}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:file_location_file_uuid' => $file_location_file_uuid, + 's2:file_location_uuid' => $file_location_uuid, + 's3:file_location_file_uuid' => $file_location_file_uuid, + 's4:file_location_active' => $file_location_active, + 's5:file_location_ready' => $file_location_ready, + 's6:file_name' => $file_name, + 's7:file_directory' => $file_directory, + 's8:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", + 's9:file_md5sum' => $file_md5sum, + }}); + + my $full_path = $file_directory."/".$file_name; + $full_path =~ s/\/\//\//g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }}); + + # If the file is not active, make sure the active is also false, regardless of anything else. + if (not $file_location_active) + { + if ($file_location_ready) + { + $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_uuid => $file_location_uuid, + file_location_file_uuid => $file_location_file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => $file_location_active, + file_location_ready => 0, + }); + } + } + elsif (-e $full_path) + { + # It exists, what's it's size? + my $real_size = (stat($full_path))[7]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + real_size => $real_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $real_size}).")", + }}); + + # If the size is the same as recorded, and the file is already 'ready', we're done. + if ($real_size == $file_size) + { + if (not $file_location_ready) + { + # Calculate the md5sum and see if it is ready now. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0265", variables => { file => $full_path }}); + if ($real_size > (128 * (2 ** 20))) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0266", variables => { + size => $anvil->Convert->bytes_to_human_readable({'bytes' => $real_size}), + }}); + } + + # Update (or get) the md5sum. + my $real_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { real_md5sum => $real_md5sum }}); + + if ($real_md5sum eq $file_md5sum) + { + # It's ready now. + $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_uuid => $file_location_uuid, + file_location_file_uuid => $file_location_file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => $file_location_active, + file_location_ready => 1, + }); + } + } + } + elsif ($file_location_ready) + { + # It's not ready. + $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_uuid => $file_location_uuid, + file_location_file_uuid => $file_location_file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => $file_location_active, + file_location_ready => 0, + }); + } + } + elsif ($file_location_ready) + { + # File doesn't exist but is marked as read, mark it as not ready. + $anvil->Database->insert_or_update_file_locations({ + debug => $debug, + file_location_uuid => $file_location_uuid, + file_location_file_uuid => $file_location_file_uuid, + file_location_host_uuid => $host_uuid, + file_location_active => $file_location_active, + file_location_ready => 0, + }); + } + } + + return(0); +} + + =head2 check_md5sums This is one half of a tool to let daemons detect when something they use has changed on disk and restart if any changes are found. @@ -3679,6 +3816,7 @@ sub push_file file_location_file_uuid => $file_uuid, file_location_host_uuid => $target_host_uuid, file_location_active => 1, + file_location_ready => "same", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); } diff --git a/notes b/notes index dc0a3bf4..a973b654 100644 --- a/notes +++ b/notes @@ -22,7 +22,7 @@ Common queries; * SELECT b.host_name, a.network_interface_uuid, a.network_interface_mac_address AS mac, a.network_interface_name AS name, a.network_interface_speed AS speed, a.network_interface_link_state AS link, a.network_interface_operational AS op, a.network_interface_duplex AS duplex, a.network_interface_medium AS medium, a.network_interface_bond_uuid AS bond_uuid, a.network_interface_bridge_uuid AS bridge_uuid FROM network_interfaces a, hosts b WHERE a.network_interface_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' AND a.network_interface_operational != 'DELETED' ORDER BY b.host_name ASC, a.network_interface_name ASC; * SELECT b.host_name, a.bond_uuid, a.bond_name, a.bond_mode, a.bond_mtu AS mtu, a.bond_primary_interface AS primary, a.bond_active_interface AS active, a.bond_mac_address AS mac, a.bond_operational AS op, c.bridge_name, a.modified_date FROM bonds a, hosts b, bridges c WHERE a.bond_host_uuid = b.host_uuid AND a.bond_bridge_uuid = c.bridge_uuid AND (b.host_uuid = 'b4e46faf-0ebe-e211-a0d6-00262d0ca874' OR b.host_uuid = '4ba42b4e-9bf7-e311-a889-899427029de4') ORDER BY b.host_name ASC, a.bond_name ASC; * SELECT b.host_name, a.bridge_uuid, a.bridge_name, a.bridge_id, a.bridge_mtu FROM bridges a, hosts b WHERE a.bridge_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' ORDER BY b.host_name ASC, a.bridge_name ASC; -* SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC; +* SELECT a.host_name, b.file_name, c.file_location_active AS active, c.file_location_ready AS ready FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC; * SELECT b.host_name, a.health_agent_name, a.health_source_name, a.health_source_weight FROM health a, hosts b WHERE b.host_uuid = a.health_host_uuid AND b.host_name LIKE 'an-a02%' order by b.host_name ASC, a.health_agent_name ASC, a.health_source_weight ASC; diff --git a/share/anvil.sql b/share/anvil.sql index 055c431b..ea6de393 100644 --- a/share/anvil.sql +++ b/share/anvil.sql @@ -1234,6 +1234,7 @@ CREATE TABLE file_locations ( file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine. file_location_host_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change. file_location_active boolean not null default TRUE, -- This is set to true when the file should be on Anvil! machines, triggering rsyncs when needed. When set to false, the file will be deleted from members, if they exist. + file_location_ready boolean not null default FALSE, -- This is set to true when the file is on the host with a good md5sum. If this is FALSE, any process needing this file should wait/loop until this goes TRUE modified_date timestamp with time zone not null, FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid), @@ -1247,6 +1248,7 @@ CREATE TABLE history.file_locations ( file_location_file_uuid text, file_location_host_uuid uuid, file_location_active boolean, + file_location_ready boolean, modified_date timestamp with time zone not null ); ALTER TABLE history.file_locations OWNER TO admin; @@ -1262,12 +1264,14 @@ BEGIN file_location_file_uuid, file_location_host_uuid, file_location_active, + file_location_ready, modified_date) VALUES (history_file_locations.file_location_uuid, history_file_locations.file_location_file_uuid, history_file_locations.file_location_host_uuid, history_file_locations.file_location_active, + history_file_locations.file_location_ready, history_file_locations.modified_date); RETURN NULL; END; diff --git a/tools/anvil-daemon b/tools/anvil-daemon index c0116114..f7af9765 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -544,39 +544,8 @@ sub handle_periodic_tasks # Scan the local network. update_state_file($anvil); - # Make sure the shared directories exist. - foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}}) - { - my $directory = $anvil->data->{path}{directories}{shared}{$target}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { - target => $target, - directory => $directory, - }}); - if (not -e $anvil->data->{path}{directories}{shared}{$target}) - { - my $failed = $anvil->Storage->make_directory({ - directory => $directory, - group => "apache", - user => "apache", - mode => "0775", - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }}); - if ($failed) - { - # Something went wrong. - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0254", variables => { - directory => $directory, - }}); - } - else - { - # Success - $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0255", variables => { - directory => $directory, - }}); - } - } - } + # Check shared files. + check_files($anvils); # Check mail server config. my $problem = $anvil->Email->check_config({debug => 3}); @@ -1703,6 +1672,52 @@ sub run_jobs return(0); } +# +sub check_files +{ + my ($anvil) = @_; + + # Make sure the shared directories exist. + foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}}) + { + my $directory = $anvil->data->{path}{directories}{shared}{$target}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + target => $target, + directory => $directory, + }}); + if (not -e $anvil->data->{path}{directories}{shared}{$target}) + { + my $failed = $anvil->Storage->make_directory({ + directory => $directory, + group => "apache", + user => "apache", + mode => "0775", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }}); + if ($failed) + { + # Something went wrong. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0254", variables => { + directory => $directory, + }}); + } + else + { + # Success + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0255", variables => { + directory => $directory, + }}); + } + } + } + + # Look for files on our system that are in file_locations. If they're shown as ready, make sure + # they're there. If they're marked as not ready, see if they now are. + $anvil->Storage->check_files({debug => 2}); + + return(0); +} + # This calls 'anvil-update-states' which will scan the local machine's state (hardware and software) and # record write it out to an HTML file sub update_state_file diff --git a/tools/anvil-manage-storage-groups b/tools/anvil-manage-storage-groups index ee8ca50a..62500ca2 100755 --- a/tools/anvil-manage-storage-groups +++ b/tools/anvil-manage-storage-groups @@ -32,8 +32,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) $| = 1; my $anvil = Anvil::Tools->new(); -$anvil->Log->level({set => 2}); -$anvil->Log->secure({set => 1}); $anvil->Get->switches({list => [ "add", diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index 6c42f8e6..ad24b2f4 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -31,8 +31,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD})) $| = 1; my $anvil = Anvil::Tools->new(); -$anvil->Log->level({set => 2}); -$anvil->Log->secure({set => 1}); # Read switches $anvil->Get->switches({list => [ diff --git a/tools/anvil-version-changes b/tools/anvil-version-changes index b303a2ac..87c1ab5a 100755 --- a/tools/anvil-version-changes +++ b/tools/anvil-version-changes @@ -77,11 +77,13 @@ sub striker_checks # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) update_audits($anvil); - ### NOTE: Disabled until review complete # This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid # entries are copied. update_dr_links($anvil); + # This checks to make sure that the new 'file_locations' -> 'file_location_ready' column exists. + update_file_location_ready($anvil); + ### TODO: Remove these later. This is here to clean up how we used to handle db_in_use and lock_request flags. if (1) { @@ -241,6 +243,101 @@ CREATE TRIGGER trigger_dr_links return(0); } +# This checks to make sure that the new 'file_locations' -> 'file_location_ready' column exists. +sub update_file_location_ready +{ + my ($anvil) = @_; + + + foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}}) + { + my $query = "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'file_locations' AND column_name = 'file_location_ready';"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); + + my $count = $anvil->Database->query({query => $query, uuid => $uuid, source => $THIS_FILE, line => __LINE__})->[0]->[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }}); + + if (not $count) + { + my $queries = []; + push @{$queries}, "ALTER TABLE public.file_locations ADD COLUMN file_location_ready boolean not null DEFAULT FALSE;"; + push @{$queries}, "ALTER TABLE history.file_locations ADD COLUMN file_location_ready boolean;"; + push @{$queries}, "DROP FUNCTION history_file_locations() CASCADE;"; + push @{$queries}, q|CREATE FUNCTION history_file_locations() RETURNS trigger +AS $$ +DECLARE + history_file_locations RECORD; +BEGIN + SELECT INTO history_file_locations * FROM file_locations WHERE file_location_uuid = new.file_location_uuid; + INSERT INTO history.file_locations + (file_location_uuid, + file_location_file_uuid, + file_location_host_uuid, + file_location_active, + file_location_ready, + modified_date) + VALUES + (history_file_locations.file_location_uuid, + history_file_locations.file_location_file_uuid, + history_file_locations.file_location_host_uuid, + history_file_locations.file_location_active, + history_file_locations.file_location_ready, + history_file_locations.modified_date); + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; +ALTER FUNCTION history_file_locations() OWNER TO admin; + +CREATE TRIGGER trigger_file_locations + AFTER INSERT OR UPDATE ON file_locations + FOR EACH ROW EXECUTE PROCEDURE history_file_locations(); +|; + foreach my $query (@{$queries}) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }}); + } + $anvil->Database->write({debug => 2, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__}); + } + } + + # Now make sure that existing DR entries are copied here. + $anvil->Database->get_hosts({deubg => 2}); + $anvil->Database->get_dr_links({debug => 2}); + + foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}}) + { + my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid}; + my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "s1:anvil_name" => $anvil_name, + "s2:anvil_uuid" => $anvil_uuid, + "s3:anvil_dr1_host_uuid" => $anvil_dr1_host_uuid, + }}); + if ($anvil_dr1_host_uuid) + { + my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{short_host_name}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }}); + + if ((not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}) or + (not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}) or + (not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}{dr_link_uuid})) + { + # Add it. + my $dr_link_uuid = $anvil->Database->insert_or_update_dr_links({ + debug => 2, + dr_link_anvil_uuid => $anvil_uuid, + dr_link_host_uuid => $anvil_dr1_host_uuid, + dr_link_note => "auto_generated", + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }}); + } + } + } + + return(0); +} + # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) sub update_audits { @@ -486,7 +583,8 @@ CREATE TRIGGER trigger_file_locations debug => 2, file_location_file_uuid => $file_uuid, file_location_host_uuid => $host_uuid, - file_location_active => 1, + file_location_active => 1, + file_location_ready => "same", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); } From 110dceb55e39c1317ca63f6443673bb293e54154 Mon Sep 17 00:00:00 2001 From: digimer Date: Thu, 4 May 2023 01:15:08 -0400 Subject: [PATCH 2/3] * Added a check to make sure files were ready before provisioning a server. Signed-off-by: digimer --- share/words.xml | 4 ++ tools/anvil-provision-server | 97 +++++++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/share/words.xml b/share/words.xml index 68080f25..e5dccfe0 100644 --- a/share/words.xml +++ b/share/words.xml @@ -3597,6 +3597,10 @@ The attempt to start the servers appears to have failed. The return code '0' was ==== We will wait: [#!variable!waiting!#] seconds and then try again. We'll give up if it keeps failing after: [#!variable!time_left!#] seconds. + [ Warning ] - The file: [#!variable!file_path!#] needed to provision the server: [#!variable!server_name!#] was not found in the database yet. + [ Warning ] - The file: [#!variable!file_path!#] needed to provision the server: [#!variable!server_name!#] was not found in the database as being on this host yet. + [ Warning ] - The file: [#!variable!file_path!#] needed to provision the server: [#!variable!server_name!#] was found, but it's not ready yet. + [ Warning ] - Waiting for a bit, and then will check if files are ready. diff --git a/tools/anvil-provision-server b/tools/anvil-provision-server index ad24b2f4..7510a30f 100755 --- a/tools/anvil-provision-server +++ b/tools/anvil-provision-server @@ -566,6 +566,101 @@ sub provision_server }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }}); + ### TODO: Left off here + # Wait until file(s) are ready. + my $waiting = 1; + my $host_uuid = $anvil->Get->host_uuid(); + my @files = ($anvil->data->{job}{install_iso_path}); + if ($anvil->data->{job}{driver_iso_path}) + { + push @files, $anvil->data->{job}{driver_iso_path}; + } + while ($waiting) + { + $anvil->Storage->check_files({debug => 2}); + $anvil->Database->get_files({debug => 2}); + $anvil->Database->get_file_locations({debug => 2}); + $waiting = 0; + foreach my $file_path (sort {$a cmp $b} @files) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_path => $file_path }}); + + $anvil->data->{file_uuid}{$file_path} = "" if not exists $anvil->data->{uuid}{$file_path}{file_uuid}; + + # What's this file's file_uuid? + my $file_uuid = $anvil->data->{file_uuid}{$file_path}; + my $file_location_uuid = ""; + if (not $file_uuid) + { + foreach my $this_file_uuid (sort {$a cmp $b} keys %{$anvil->data->{files}{file_uuid}}) + { + my $this_file_path = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$this_file_uuid}{file_name}; + $this_file_path =~ s/\/\//\//g; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + this_file_uuid => $this_file_uuid, + this_file_path => $this_file_path, + }}); + if ($this_file_path eq $file_path) + { + # Found it. + $file_uuid = $this_file_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }}); + } + } + } + + # Did we find the file? + if (not $file_uuid) + { + # Nope. + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0154", variables => { + file_path => $file_path, + server_name => $server, + }}); + next; + } + + # Yes, now do we have the file_location_uuid? + if (not exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}) + { + # Nope + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0155", variables => { + file_path => $file_path, + server_name => $server, + }}); + next; + } + + $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid}; + my $file_location_ready = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + file_location_uuid => $file_location_uuid, + file_location_ready => $file_location_ready, + }}); + + if (not $file_location_ready) + { + $waiting = 1; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }}); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0156", variables => { + file_path => $file_path, + server_name => $server, + }}); + next; + } + } + + if ($waiting) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0157"}); + sleep 10; + } + } + # Call as a background process. my ($handle, $return_code) = $anvil->System->call({ background => 1, @@ -587,7 +682,7 @@ sub provision_server # Loop for up to 10 seconds waiting to see the server start running. my $wait_until = time + 60; - my $waiting = 1; + $waiting = 1; my $status = ""; while($waiting) { From 8f375c58a932e11af9aac8b419f1d7685980d2cf Mon Sep 17 00:00:00 2001 From: digimer Date: Thu, 4 May 2023 11:14:23 -0400 Subject: [PATCH 3/3] * Fixed a typo in anvil-daemon that prevented compiling. Signed-off-by: digimer --- tools/anvil-daemon | 2 +- tools/striker-prep-database | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/anvil-daemon b/tools/anvil-daemon index f7af9765..c2cec9ec 100755 --- a/tools/anvil-daemon +++ b/tools/anvil-daemon @@ -545,7 +545,7 @@ sub handle_periodic_tasks update_state_file($anvil); # Check shared files. - check_files($anvils); + check_files($anvil); # Check mail server config. my $problem = $anvil->Email->check_config({debug => 3}); diff --git a/tools/striker-prep-database b/tools/striker-prep-database index f049ec5b..f39db754 100755 --- a/tools/striker-prep-database +++ b/tools/striker-prep-database @@ -462,7 +462,7 @@ if ($local_uuid) # In some cases, the database won't allow connections to the admin user. To deal with this, we'll # call stop->start on the daemon (reload doesn't fix it). - my $return_code = $anvil->System->stop_daemon({daemon => "postgresql"}); + $return_code = $anvil->System->stop_daemon({daemon => "postgresql"}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }}); $return_code = $anvil->System->start_daemon({daemon => "postgresql"});