Merge pull request #328 from ClusterLabs/anvil-tools-dev

This should resolve issue #271.
main
Digimer 2 years ago committed by GitHub
commit f3a65fc04d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 55
      Anvil/Tools/Database.pm
  2. 138
      Anvil/Tools/Storage.pm
  3. 2
      notes
  4. 4
      share/anvil.sql
  5. 4
      share/words.xml
  6. 81
      tools/anvil-daemon
  7. 2
      tools/anvil-manage-storage-groups
  8. 99
      tools/anvil-provision-server
  9. 100
      tools/anvil-version-changes
  10. 2
      tools/striker-prep-database

@ -429,6 +429,7 @@ sub check_file_locations
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid, file_location_host_uuid => $host_uuid,
file_location_active => 1, file_location_active => 1,
file_location_ready => "same",
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
} }
@ -3487,6 +3488,7 @@ This loads the known install file_locations into the C<< anvil::data >> hash at:
* file_locations::file_location_uuid::<file_location_uuid>::file_location_file_uuid * file_locations::file_location_uuid::<file_location_uuid>::file_location_file_uuid
* file_locations::file_location_uuid::<file_location_uuid>::file_location_host_uuid * file_locations::file_location_uuid::<file_location_uuid>::file_location_host_uuid
* file_locations::file_location_uuid::<file_location_uuid>::file_location_active * file_locations::file_location_uuid::<file_location_uuid>::file_location_active
* file_locations::file_location_uuid::<file_location_uuid>::file_location_ready
* file_locations::file_location_uuid::<file_location_uuid>::modified_date * file_locations::file_location_uuid::<file_location_uuid>::modified_date
If the hash was already populated, it is cleared before repopulating to ensure no stale data remains. If the hash was already populated, it is cleared before repopulating to ensure no stale data remains.
@ -3513,6 +3515,7 @@ SELECT
file_location_file_uuid, file_location_file_uuid,
file_location_host_uuid, file_location_host_uuid,
file_location_active, file_location_active,
file_location_ready,
modified_date modified_date
FROM FROM
file_locations file_locations
@ -3530,12 +3533,14 @@ FROM
my $file_location_file_uuid = $row->[1]; my $file_location_file_uuid = $row->[1];
my $file_location_host_uuid = $row->[2]; my $file_location_host_uuid = $row->[2];
my $file_location_active = $row->[3]; my $file_location_active = $row->[3];
my $modified_date = $row->[4]; my $file_location_ready = $row->[4];
my $modified_date = $row->[5];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
file_location_uuid => $file_location_uuid, file_location_uuid => $file_location_uuid,
file_location_file_uuid => $file_location_file_uuid, file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $file_location_host_uuid, file_location_host_uuid => $file_location_host_uuid,
file_location_active => $file_location_active, file_location_active => $file_location_active,
file_location_ready => $file_location_ready,
modified_date => $modified_date, modified_date => $modified_date,
}}); }});
@ -3543,11 +3548,13 @@ FROM
$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid} = $file_location_file_uuid; $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid} = $file_location_file_uuid;
$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid} = $file_location_host_uuid; $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid} = $file_location_host_uuid;
$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active} = $file_location_active; $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active} = $file_location_active;
$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready} = $file_location_ready;
$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date} = $modified_date; $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date} = $modified_date;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"file_locations::file_location_uuid::${file_location_uuid}::file_location_file_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid}, "file_locations::file_location_uuid::${file_location_uuid}::file_location_file_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid},
"file_locations::file_location_uuid::${file_location_uuid}::file_location_host_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid}, "file_locations::file_location_uuid::${file_location_uuid}::file_location_host_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_host_uuid},
"file_locations::file_location_uuid::${file_location_uuid}::file_location_active" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active}, "file_locations::file_location_uuid::${file_location_uuid}::file_location_active" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active},
"file_locations::file_location_uuid::${file_location_uuid}::file_location_ready" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready},
"file_locations::file_location_uuid::${file_location_uuid}::modified_date" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date}, "file_locations::file_location_uuid::${file_location_uuid}::modified_date" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date},
}}); }});
@ -8490,6 +8497,14 @@ This is set to C<< 1 >> or C<< 0 >>, and indicates if the file should be on the
When set to C<< 1 >>, the file will be copied by the Anvil! member machines (by the member machines, they pull the files using rsync). If set to C<< 0 >>, the file is marked as inactive. If the file exists on the Anvil! members, it will be deleted. When set to C<< 1 >>, the file will be copied by the Anvil! member machines (by the member machines, they pull the files using rsync). If set to C<< 0 >>, the file is marked as inactive. If the file exists on the Anvil! members, it will be deleted.
=head3 file_location_ready (optional, default '0')
This is set to C<< 1 >> or C<< 0 >>, and indicates if the file is on the system and ready to be used.
B<< Note >>: This can also be set to C<< same >>. If set, and the file exists in the database, the existing value is retained. If the entry is inserted, this is set to C<< 0 >>.
When set to C<< 1 >>, the file's size and md5sum have been confirmed to match on disk what is recorded in the database. When set to C<< 0 >>, the file _may_ be ready, but it probably isn't yet. Any process needing the file should check that it's ready before using it.
=cut =cut
sub insert_or_update_file_locations sub insert_or_update_file_locations
{ {
@ -8506,7 +8521,8 @@ sub insert_or_update_file_locations
my $file_location_anvil_uuid = defined $parameter->{file_location_anvil_uuid} ? $parameter->{file_location_anvil_uuid} : ""; my $file_location_anvil_uuid = defined $parameter->{file_location_anvil_uuid} ? $parameter->{file_location_anvil_uuid} : "";
my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : ""; my $file_location_file_uuid = defined $parameter->{file_location_file_uuid} ? $parameter->{file_location_file_uuid} : "";
my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : ""; my $file_location_host_uuid = defined $parameter->{file_location_host_uuid} ? $parameter->{file_location_host_uuid} : "";
my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : ""; my $file_location_active = defined $parameter->{file_location_active} ? $parameter->{file_location_active} : 0;
my $file_location_ready = defined $parameter->{file_location_ready} ? $parameter->{file_location_ready} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
uuid => $uuid, uuid => $uuid,
file => $file, file => $file,
@ -8516,6 +8532,7 @@ sub insert_or_update_file_locations
file_location_file_uuid => $file_location_file_uuid, file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $file_location_host_uuid, file_location_host_uuid => $file_location_host_uuid,
file_location_active => $file_location_active, file_location_active => $file_location_active,
file_location_ready => $file_location_ready,
}}); }});
if (not $file_location_file_uuid) if (not $file_location_file_uuid)
@ -8536,6 +8553,12 @@ sub insert_or_update_file_locations
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_active" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_active" }});
return(""); return("");
} }
if (($file_location_ready ne "0") && ($file_location_ready ne "1") && ($file_location_ready ne "same"))
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->insert_or_update_file_locations()", parameter => "file_location_ready" }});
return("");
}
# If we've got an Anvil! uuid, find out the hosts and DR links connected to the Anvil! are found and # If we've got an Anvil! uuid, find out the hosts and DR links connected to the Anvil! are found and
# this method is recursively called for each host. # this method is recursively called for each host.
@ -8577,6 +8600,7 @@ sub insert_or_update_file_locations
file_location_file_uuid => $file_location_file_uuid, file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $host_uuid, file_location_host_uuid => $host_uuid,
file_location_active => $file_location_active, file_location_active => $file_location_active,
file_location_ready => $file_location_ready,
}); });
$file_location_uuids .= $host_uuid."=".$file_location_uuid.","; $file_location_uuids .= $host_uuid."=".$file_location_uuid.",";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
@ -8626,6 +8650,12 @@ AND
$file_location_uuid = $anvil->Get->uuid(); $file_location_uuid = $anvil->Get->uuid();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
if ($file_location_ready eq "same")
{
$file_location_ready = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_ready => $file_location_ready }});
}
my $query = " my $query = "
INSERT INTO INSERT INTO
file_locations file_locations
@ -8634,12 +8664,14 @@ INSERT INTO
file_location_file_uuid, file_location_file_uuid,
file_location_host_uuid, file_location_host_uuid,
file_location_active, file_location_active,
file_location_ready,
modified_date modified_date
) VALUES ( ) VALUES (
".$anvil->Database->quote($file_location_uuid).", ".$anvil->Database->quote($file_location_uuid).",
".$anvil->Database->quote($file_location_file_uuid).", ".$anvil->Database->quote($file_location_file_uuid).",
".$anvil->Database->quote($file_location_host_uuid).", ".$anvil->Database->quote($file_location_host_uuid).",
".$anvil->Database->quote($file_location_active).", ".$anvil->Database->quote($file_location_active).",
".$anvil->Database->quote($file_location_ready).",
".$anvil->Database->quote($anvil->Database->refresh_timestamp)." ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
); );
"; ";
@ -8653,7 +8685,8 @@ INSERT INTO
SELECT SELECT
file_location_file_uuid, file_location_file_uuid,
file_location_host_uuid, file_location_host_uuid,
file_location_active file_location_active,
file_location_ready
FROM FROM
file_locations file_locations
WHERE WHERE
@ -8678,16 +8711,25 @@ WHERE
my $old_file_location_file_uuid = $row->[0]; my $old_file_location_file_uuid = $row->[0];
my $old_file_location_host_uuid = $row->[1]; my $old_file_location_host_uuid = $row->[1];
my $old_file_location_active = $row->[2]; my $old_file_location_active = $row->[2];
my $old_file_location_ready = $row->[3];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
old_file_location_file_uuid => $old_file_location_file_uuid, old_file_location_file_uuid => $old_file_location_file_uuid,
old_file_location_host_uuid => $old_file_location_host_uuid, old_file_location_host_uuid => $old_file_location_host_uuid,
old_file_location_active => $old_file_location_active, old_file_location_active => $old_file_location_active,
old_file_location_ready => $old_file_location_ready,
}}); }});
if ($file_location_ready eq "same")
{
$file_location_ready = $old_file_location_ready;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_ready => $file_location_ready }});
}
# Anything change? # Anything change?
if (($old_file_location_file_uuid ne $file_location_file_uuid) or if (($old_file_location_file_uuid ne $file_location_file_uuid) or
($old_file_location_host_uuid ne $file_location_host_uuid) or ($old_file_location_host_uuid ne $file_location_host_uuid) or
($old_file_location_active ne $file_location_active)) ($old_file_location_active ne $file_location_active) or
($old_file_location_ready ne $file_location_ready))
{ {
# Something changed, save. # Something changed, save.
my $query = " my $query = "
@ -8697,6 +8739,7 @@ SET
file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid).", file_location_file_uuid = ".$anvil->Database->quote($file_location_file_uuid).",
file_location_host_uuid = ".$anvil->Database->quote($file_location_host_uuid).", file_location_host_uuid = ".$anvil->Database->quote($file_location_host_uuid).",
file_location_active = ".$anvil->Database->quote($file_location_active).", file_location_active = ".$anvil->Database->quote($file_location_active).",
file_location_ready = ".$anvil->Database->quote($file_location_ready).",
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)." modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE WHERE
file_location_uuid = ".$anvil->Database->quote($file_location_uuid)." file_location_uuid = ".$anvil->Database->quote($file_location_uuid)."
@ -18185,6 +18228,7 @@ sub track_files
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid, file_location_host_uuid => $host_uuid,
file_location_active => 1, file_location_active => 1,
file_location_ready => "same",
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
reload => $reload, reload => $reload,
@ -18250,6 +18294,7 @@ sub track_files
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid, file_location_host_uuid => $host_uuid,
file_location_active => 1, file_location_active => 1,
file_location_ready => "same",
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
} }
@ -18267,6 +18312,7 @@ sub track_files
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid, file_location_host_uuid => $host_uuid,
file_location_active => 1, file_location_active => 1,
file_location_ready => "same",
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
} }
@ -18323,6 +18369,7 @@ sub track_files
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid, file_location_host_uuid => $host_uuid,
file_location_active => 1, file_location_active => 1,
file_location_ready => "same",
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_location_uuid => $file_location_uuid }});
} }

@ -19,6 +19,7 @@ my $THIS_FILE = "Storage.pm";
# backup # backup
# change_mode # change_mode
# change_owner # change_owner
# check_files
# check_md5sums # check_md5sums
# compress # compress
# copy_file # copy_file
@ -583,6 +584,142 @@ sub change_owner
return($error); return($error);
} }
=head2 check_files
This method checks the files on the local system. Specifically, it looks in C<< file_locations >> table and then checks if the file is "ready" or not. Depending on the results, C<< file_location_ready >> is updated if needed.
This method takes no parameters.
=cut
sub check_files
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->check_md5sums()" }});
$anvil->Database->get_files({debug => $debug});
$anvil->Database->get_file_locations({debug => $debug});
# Look for files that should be on this host.
my $host_uuid = $anvil->Get->host_uuid({debug => $debug});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
# Sorting isn't useful really, but it ensures consistent listing run over run).
foreach my $file_location_file_uuid (sort {$a cmp $b} keys %{$anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}})
{
my $file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_location_file_uuid}{file_location_uuid};
my $file_location_file_uuid = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_file_uuid};
my $file_location_active = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_active};
my $file_location_ready = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready};
my $file_name = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_name};
my $file_directory = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_directory};
my $file_size = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_size};
my $file_md5sum = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_md5sum};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:file_location_file_uuid' => $file_location_file_uuid,
's2:file_location_uuid' => $file_location_uuid,
's3:file_location_file_uuid' => $file_location_file_uuid,
's4:file_location_active' => $file_location_active,
's5:file_location_ready' => $file_location_ready,
's6:file_name' => $file_name,
's7:file_directory' => $file_directory,
's8:file_size' => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")",
's9:file_md5sum' => $file_md5sum,
}});
my $full_path = $file_directory."/".$file_name;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }});
# If the file is not active, make sure the active is also false, regardless of anything else.
if (not $file_location_active)
{
if ($file_location_ready)
{
$anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_uuid => $file_location_uuid,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => $file_location_active,
file_location_ready => 0,
});
}
}
elsif (-e $full_path)
{
# It exists, what's it's size?
my $real_size = (stat($full_path))[7];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
real_size => $real_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $real_size}).")",
}});
# If the size is the same as recorded, and the file is already 'ready', we're done.
if ($real_size == $file_size)
{
if (not $file_location_ready)
{
# Calculate the md5sum and see if it is ready now.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0265", variables => { file => $full_path }});
if ($real_size > (128 * (2 ** 20)))
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0266", variables => {
size => $anvil->Convert->bytes_to_human_readable({'bytes' => $real_size}),
}});
}
# Update (or get) the md5sum.
my $real_md5sum = $anvil->Get->md5sum({debug => 2, file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { real_md5sum => $real_md5sum }});
if ($real_md5sum eq $file_md5sum)
{
# It's ready now.
$anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_uuid => $file_location_uuid,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => $file_location_active,
file_location_ready => 1,
});
}
}
}
elsif ($file_location_ready)
{
# It's not ready.
$anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_uuid => $file_location_uuid,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => $file_location_active,
file_location_ready => 0,
});
}
}
elsif ($file_location_ready)
{
# File doesn't exist but is marked as read, mark it as not ready.
$anvil->Database->insert_or_update_file_locations({
debug => $debug,
file_location_uuid => $file_location_uuid,
file_location_file_uuid => $file_location_file_uuid,
file_location_host_uuid => $host_uuid,
file_location_active => $file_location_active,
file_location_ready => 0,
});
}
}
return(0);
}
=head2 check_md5sums =head2 check_md5sums
This is one half of a tool to let daemons detect when something they use has changed on disk and restart if any changes are found. This is one half of a tool to let daemons detect when something they use has changed on disk and restart if any changes are found.
@ -3679,6 +3816,7 @@ sub push_file
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_host_uuid => $target_host_uuid, file_location_host_uuid => $target_host_uuid,
file_location_active => 1, file_location_active => 1,
file_location_ready => "same",
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
} }

@ -22,7 +22,7 @@ Common queries;
* SELECT b.host_name, a.network_interface_uuid, a.network_interface_mac_address AS mac, a.network_interface_name AS name, a.network_interface_speed AS speed, a.network_interface_link_state AS link, a.network_interface_operational AS op, a.network_interface_duplex AS duplex, a.network_interface_medium AS medium, a.network_interface_bond_uuid AS bond_uuid, a.network_interface_bridge_uuid AS bridge_uuid FROM network_interfaces a, hosts b WHERE a.network_interface_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' AND a.network_interface_operational != 'DELETED' ORDER BY b.host_name ASC, a.network_interface_name ASC; * SELECT b.host_name, a.network_interface_uuid, a.network_interface_mac_address AS mac, a.network_interface_name AS name, a.network_interface_speed AS speed, a.network_interface_link_state AS link, a.network_interface_operational AS op, a.network_interface_duplex AS duplex, a.network_interface_medium AS medium, a.network_interface_bond_uuid AS bond_uuid, a.network_interface_bridge_uuid AS bridge_uuid FROM network_interfaces a, hosts b WHERE a.network_interface_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' AND a.network_interface_operational != 'DELETED' ORDER BY b.host_name ASC, a.network_interface_name ASC;
* SELECT b.host_name, a.bond_uuid, a.bond_name, a.bond_mode, a.bond_mtu AS mtu, a.bond_primary_interface AS primary, a.bond_active_interface AS active, a.bond_mac_address AS mac, a.bond_operational AS op, c.bridge_name, a.modified_date FROM bonds a, hosts b, bridges c WHERE a.bond_host_uuid = b.host_uuid AND a.bond_bridge_uuid = c.bridge_uuid AND (b.host_uuid = 'b4e46faf-0ebe-e211-a0d6-00262d0ca874' OR b.host_uuid = '4ba42b4e-9bf7-e311-a889-899427029de4') ORDER BY b.host_name ASC, a.bond_name ASC; * SELECT b.host_name, a.bond_uuid, a.bond_name, a.bond_mode, a.bond_mtu AS mtu, a.bond_primary_interface AS primary, a.bond_active_interface AS active, a.bond_mac_address AS mac, a.bond_operational AS op, c.bridge_name, a.modified_date FROM bonds a, hosts b, bridges c WHERE a.bond_host_uuid = b.host_uuid AND a.bond_bridge_uuid = c.bridge_uuid AND (b.host_uuid = 'b4e46faf-0ebe-e211-a0d6-00262d0ca874' OR b.host_uuid = '4ba42b4e-9bf7-e311-a889-899427029de4') ORDER BY b.host_name ASC, a.bond_name ASC;
* SELECT b.host_name, a.bridge_uuid, a.bridge_name, a.bridge_id, a.bridge_mtu FROM bridges a, hosts b WHERE a.bridge_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' ORDER BY b.host_name ASC, a.bridge_name ASC; * SELECT b.host_name, a.bridge_uuid, a.bridge_name, a.bridge_id, a.bridge_mtu FROM bridges a, hosts b WHERE a.bridge_host_uuid = b.host_uuid AND b.host_name LIKE 'an-a02%' ORDER BY b.host_name ASC, a.bridge_name ASC;
* SELECT a.host_name, b.file_name, c.file_location_active FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC; * SELECT a.host_name, b.file_name, c.file_location_active AS active, c.file_location_ready AS ready FROM hosts a, files b, file_locations c WHERE a.host_uuid = c.file_location_host_uuid AND b.file_uuid = c.file_location_file_uuid ORDER BY b.file_name ASC, a.host_name ASC;
* SELECT b.host_name, a.health_agent_name, a.health_source_name, a.health_source_weight FROM health a, hosts b WHERE b.host_uuid = a.health_host_uuid AND b.host_name LIKE 'an-a02%' order by b.host_name ASC, a.health_agent_name ASC, a.health_source_weight ASC; * SELECT b.host_name, a.health_agent_name, a.health_source_name, a.health_source_weight FROM health a, hosts b WHERE b.host_uuid = a.health_host_uuid AND b.host_name LIKE 'an-a02%' order by b.host_name ASC, a.health_agent_name ASC, a.health_source_weight ASC;

@ -1234,6 +1234,7 @@ CREATE TABLE file_locations (
file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine. file_location_file_uuid uuid not null, -- This is file to be moved to (or restored to) this machine.
file_location_host_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change. file_location_host_uuid uuid not null, -- This is the sum as calculated when the file_location is first uploaded. Once recorded, it can't change.
file_location_active boolean not null default TRUE, -- This is set to true when the file should be on Anvil! machines, triggering rsyncs when needed. When set to false, the file will be deleted from members, if they exist. file_location_active boolean not null default TRUE, -- This is set to true when the file should be on Anvil! machines, triggering rsyncs when needed. When set to false, the file will be deleted from members, if they exist.
file_location_ready boolean not null default FALSE, -- This is set to true when the file is on the host with a good md5sum. If this is FALSE, any process needing this file should wait/loop until this goes TRUE
modified_date timestamp with time zone not null, modified_date timestamp with time zone not null,
FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid), FOREIGN KEY(file_location_file_uuid) REFERENCES files(file_uuid),
@ -1247,6 +1248,7 @@ CREATE TABLE history.file_locations (
file_location_file_uuid text, file_location_file_uuid text,
file_location_host_uuid uuid, file_location_host_uuid uuid,
file_location_active boolean, file_location_active boolean,
file_location_ready boolean,
modified_date timestamp with time zone not null modified_date timestamp with time zone not null
); );
ALTER TABLE history.file_locations OWNER TO admin; ALTER TABLE history.file_locations OWNER TO admin;
@ -1262,12 +1264,14 @@ BEGIN
file_location_file_uuid, file_location_file_uuid,
file_location_host_uuid, file_location_host_uuid,
file_location_active, file_location_active,
file_location_ready,
modified_date) modified_date)
VALUES VALUES
(history_file_locations.file_location_uuid, (history_file_locations.file_location_uuid,
history_file_locations.file_location_file_uuid, history_file_locations.file_location_file_uuid,
history_file_locations.file_location_host_uuid, history_file_locations.file_location_host_uuid,
history_file_locations.file_location_active, history_file_locations.file_location_active,
history_file_locations.file_location_ready,
history_file_locations.modified_date); history_file_locations.modified_date);
RETURN NULL; RETURN NULL;
END; END;

@ -3597,6 +3597,10 @@ The attempt to start the servers appears to have failed. The return code '0' was
==== ====
We will wait: [#!variable!waiting!#] seconds and then try again. We'll give up if it keeps failing after: [#!variable!time_left!#] seconds. We will wait: [#!variable!waiting!#] seconds and then try again. We'll give up if it keeps failing after: [#!variable!time_left!#] seconds.
</key> </key>
<key name="warning_0154">[ Warning ] - The file: [#!variable!file_path!#] needed to provision the server: [#!variable!server_name!#] was not found in the database yet.</key>
<key name="warning_0155">[ Warning ] - The file: [#!variable!file_path!#] needed to provision the server: [#!variable!server_name!#] was not found in the database as being on this host yet.</key>
<key name="warning_0156">[ Warning ] - The file: [#!variable!file_path!#] needed to provision the server: [#!variable!server_name!#] was found, but it's not ready yet.</key>
<key name="warning_0157">[ Warning ] - Waiting for a bit, and then will check if files are ready.</key>
</language> </language>
<!-- 日本語 --> <!-- 日本語 -->

@ -544,39 +544,8 @@ sub handle_periodic_tasks
# Scan the local network. # Scan the local network.
update_state_file($anvil); update_state_file($anvil);
# Make sure the shared directories exist. # Check shared files.
foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}}) check_files($anvil);
{
my $directory = $anvil->data->{path}{directories}{shared}{$target};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
target => $target,
directory => $directory,
}});
if (not -e $anvil->data->{path}{directories}{shared}{$target})
{
my $failed = $anvil->Storage->make_directory({
directory => $directory,
group => "apache",
user => "apache",
mode => "0775",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
if ($failed)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0254", variables => {
directory => $directory,
}});
}
else
{
# Success
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0255", variables => {
directory => $directory,
}});
}
}
}
# Check mail server config. # Check mail server config.
my $problem = $anvil->Email->check_config({debug => 3}); my $problem = $anvil->Email->check_config({debug => 3});
@ -1703,6 +1672,52 @@ sub run_jobs
return(0); return(0);
} }
#
sub check_files
{
my ($anvil) = @_;
# Make sure the shared directories exist.
foreach my $target (sort {$a cmp $b} keys %{$anvil->data->{path}{directories}{shared}})
{
my $directory = $anvil->data->{path}{directories}{shared}{$target};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
target => $target,
directory => $directory,
}});
if (not -e $anvil->data->{path}{directories}{shared}{$target})
{
my $failed = $anvil->Storage->make_directory({
directory => $directory,
group => "apache",
user => "apache",
mode => "0775",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { failed => $failed }});
if ($failed)
{
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "log_0254", variables => {
directory => $directory,
}});
}
else
{
# Success
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0255", variables => {
directory => $directory,
}});
}
}
}
# Look for files on our system that are in file_locations. If they're shown as ready, make sure
# they're there. If they're marked as not ready, see if they now are.
$anvil->Storage->check_files({debug => 2});
return(0);
}
# This calls 'anvil-update-states' which will scan the local machine's state (hardware and software) and # This calls 'anvil-update-states' which will scan the local machine's state (hardware and software) and
# record write it out to an HTML file # record write it out to an HTML file
sub update_state_file sub update_state_file

@ -32,8 +32,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
$| = 1; $| = 1;
my $anvil = Anvil::Tools->new(); my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Get->switches({list => [ $anvil->Get->switches({list => [
"add", "add",

@ -31,8 +31,6 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
$| = 1; $| = 1;
my $anvil = Anvil::Tools->new(); my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
# Read switches # Read switches
$anvil->Get->switches({list => [ $anvil->Get->switches({list => [
@ -568,6 +566,101 @@ sub provision_server
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { variable_uuid => $variable_uuid }});
### TODO: Left off here
# Wait until file(s) are ready.
my $waiting = 1;
my $host_uuid = $anvil->Get->host_uuid();
my @files = ($anvil->data->{job}{install_iso_path});
if ($anvil->data->{job}{driver_iso_path})
{
push @files, $anvil->data->{job}{driver_iso_path};
}
while ($waiting)
{
$anvil->Storage->check_files({debug => 2});
$anvil->Database->get_files({debug => 2});
$anvil->Database->get_file_locations({debug => 2});
$waiting = 0;
foreach my $file_path (sort {$a cmp $b} @files)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_path => $file_path }});
$anvil->data->{file_uuid}{$file_path} = "" if not exists $anvil->data->{uuid}{$file_path}{file_uuid};
# What's this file's file_uuid?
my $file_uuid = $anvil->data->{file_uuid}{$file_path};
my $file_location_uuid = "";
if (not $file_uuid)
{
foreach my $this_file_uuid (sort {$a cmp $b} keys %{$anvil->data->{files}{file_uuid}})
{
my $this_file_path = $anvil->data->{files}{file_uuid}{$this_file_uuid}{file_directory}."/".$anvil->data->{files}{file_uuid}{$this_file_uuid}{file_name};
$this_file_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
this_file_uuid => $this_file_uuid,
this_file_path => $this_file_path,
}});
if ($this_file_path eq $file_path)
{
# Found it.
$file_uuid = $this_file_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_uuid => $file_uuid }});
}
}
}
# Did we find the file?
if (not $file_uuid)
{
# Nope.
$waiting = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0154", variables => {
file_path => $file_path,
server_name => $server,
}});
next;
}
# Yes, now do we have the file_location_uuid?
if (not exists $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid})
{
# Nope
$waiting = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0155", variables => {
file_path => $file_path,
server_name => $server,
}});
next;
}
$file_location_uuid = $anvil->data->{file_locations}{host_uuid}{$host_uuid}{file_uuid}{$file_uuid}{file_location_uuid};
my $file_location_ready = $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_ready};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
file_location_uuid => $file_location_uuid,
file_location_ready => $file_location_ready,
}});
if (not $file_location_ready)
{
$waiting = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0156", variables => {
file_path => $file_path,
server_name => $server,
}});
next;
}
}
if ($waiting)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "alert", key => "warning_0157"});
sleep 10;
}
}
# Call as a background process. # Call as a background process.
my ($handle, $return_code) = $anvil->System->call({ my ($handle, $return_code) = $anvil->System->call({
background => 1, background => 1,
@ -589,7 +682,7 @@ sub provision_server
# Loop for up to 10 seconds waiting to see the server start running. # Loop for up to 10 seconds waiting to see the server start running.
my $wait_until = time + 60; my $wait_until = time + 60;
my $waiting = 1; $waiting = 1;
my $status = ""; my $status = "";
while($waiting) while($waiting)
{ {

@ -77,11 +77,13 @@ sub striker_checks
# This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release)
update_audits($anvil); update_audits($anvil);
### NOTE: Disabled until review complete
# This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid # This checks to make sure that the new dr_links table exists, and that existing anvil_dr1_host_uuid
# entries are copied. # entries are copied.
update_dr_links($anvil); update_dr_links($anvil);
# This checks to make sure that the new 'file_locations' -> 'file_location_ready' column exists.
update_file_location_ready($anvil);
### TODO: Remove these later. This is here to clean up how we used to handle db_in_use and lock_request flags. ### TODO: Remove these later. This is here to clean up how we used to handle db_in_use and lock_request flags.
if (1) if (1)
{ {
@ -241,6 +243,101 @@ CREATE TRIGGER trigger_dr_links
return(0); return(0);
} }
# This checks to make sure that the new 'file_locations' -> 'file_location_ready' column exists.
sub update_file_location_ready
{
my ($anvil) = @_;
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{cache}{database_handle}})
{
my $query = "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'file_locations' AND column_name = 'file_location_ready';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $count = $anvil->Database->query({query => $query, uuid => $uuid, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
if (not $count)
{
my $queries = [];
push @{$queries}, "ALTER TABLE public.file_locations ADD COLUMN file_location_ready boolean not null DEFAULT FALSE;";
push @{$queries}, "ALTER TABLE history.file_locations ADD COLUMN file_location_ready boolean;";
push @{$queries}, "DROP FUNCTION history_file_locations() CASCADE;";
push @{$queries}, q|CREATE FUNCTION history_file_locations() RETURNS trigger
AS $$
DECLARE
history_file_locations RECORD;
BEGIN
SELECT INTO history_file_locations * FROM file_locations WHERE file_location_uuid = new.file_location_uuid;
INSERT INTO history.file_locations
(file_location_uuid,
file_location_file_uuid,
file_location_host_uuid,
file_location_active,
file_location_ready,
modified_date)
VALUES
(history_file_locations.file_location_uuid,
history_file_locations.file_location_file_uuid,
history_file_locations.file_location_host_uuid,
history_file_locations.file_location_active,
history_file_locations.file_location_ready,
history_file_locations.modified_date);
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
ALTER FUNCTION history_file_locations() OWNER TO admin;
CREATE TRIGGER trigger_file_locations
AFTER INSERT OR UPDATE ON file_locations
FOR EACH ROW EXECUTE PROCEDURE history_file_locations();
|;
foreach my $query (@{$queries})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
}
$anvil->Database->write({debug => 2, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__});
}
}
# Now make sure that existing DR entries are copied here.
$anvil->Database->get_hosts({deubg => 2});
$anvil->Database->get_dr_links({debug => 2});
foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}})
{
my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid};
my $anvil_dr1_host_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_dr1_host_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:anvil_name" => $anvil_name,
"s2:anvil_uuid" => $anvil_uuid,
"s3:anvil_dr1_host_uuid" => $anvil_dr1_host_uuid,
}});
if ($anvil_dr1_host_uuid)
{
my $dr1_host_name = $anvil->data->{hosts}{host_uuid}{$anvil_dr1_host_uuid}{short_host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }});
if ((not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}) or
(not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}) or
(not exists $anvil->data->{dr_links}{by_anvil_uuid}{$anvil_uuid}{dr_link_host_uuid}{$anvil_dr1_host_uuid}{dr_link_uuid}))
{
# Add it.
my $dr_link_uuid = $anvil->Database->insert_or_update_dr_links({
debug => 2,
dr_link_anvil_uuid => $anvil_uuid,
dr_link_host_uuid => $anvil_dr1_host_uuid,
dr_link_note => "auto_generated",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr1_host_name => $dr1_host_name }});
}
}
}
return(0);
}
# This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release) # This checks to make sure that the 'audits' table exists (added late into M3.0 pre-release)
sub update_audits sub update_audits
{ {
@ -487,6 +584,7 @@ CREATE TRIGGER trigger_file_locations
file_location_file_uuid => $file_uuid, file_location_file_uuid => $file_uuid,
file_location_host_uuid => $host_uuid, file_location_host_uuid => $host_uuid,
file_location_active => 1, file_location_active => 1,
file_location_ready => "same",
}); });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { file_location_uuid => $file_location_uuid }});
} }

@ -462,7 +462,7 @@ if ($local_uuid)
# In some cases, the database won't allow connections to the admin user. To deal with this, we'll # In some cases, the database won't allow connections to the admin user. To deal with this, we'll
# call stop->start on the daemon (reload doesn't fix it). # call stop->start on the daemon (reload doesn't fix it).
my $return_code = $anvil->System->stop_daemon({daemon => "postgresql"}); $return_code = $anvil->System->stop_daemon({daemon => "postgresql"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { return_code => $return_code }});
$return_code = $anvil->System->start_daemon({daemon => "postgresql"}); $return_code = $anvil->System->start_daemon({daemon => "postgresql"});

Loading…
Cancel
Save