seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Fixed a bug in anvil-daemon where completed jobs could be immediately cleared, causing them to re-run (repeatedly).

Browse Source 
* Added 'sys::log_date' which controls if the date and time is pre-pended to log entries.
* Created Get->host_name() which takes a host UUID and returns the 'host_name' from the 'hosts' table, if found.
* Cleaned up some HTML templates and logging.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 6 years ago
parent e15fd19ee4
commit facefeaccc
10 changed files with 127 additions and 33 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      Anvil/Tools.pm
  2. 59
      Anvil/Tools/Database.pm
  3. 48
      Anvil/Tools/Get.pm
  4. 5
      Anvil/Tools/Log.pm
  5. 14
      cgi-bin/striker
  6. 2
      html/skins/alteeve/main.css
  7. 4
      html/skins/alteeve/striker.html
  8. 5
      share/words.xml
  9. 8
      tools/anvil-daemon
  10. 8
      tools/anvil-manage-striker-peers

1
Anvil/Tools.pm
Unescape View File

@ -783,6 +783,7 @@ sub _set_defaults
host_type => "",
host_uuid => "",
language => "en_CA",
log_date => 1,
log_file => "/var/log/anvil.log",
password => {
algorithm => "sha512",

59
Anvil/Tools/Database.pm
Unescape View File

@ -1363,8 +1363,9 @@ WHERE
my $job_description = $row->[9];
my $job_status = $row->[10];
my $modified_date = $row->[11];
my $started_seconds_ago = $job_picked_up_at ? (time - $job_picked_up_at) : 0;
my $updated_seconds_ago = $job_updated ? (time - $job_updated) : 0;
my $now_time = time;
my $started_seconds_ago = $job_picked_up_at ? ($now_time - $job_picked_up_at) : 0;
my $updated_seconds_ago = $job_updated ? ($now_time - $job_updated) : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
job_uuid => $job_uuid,
job_command => $job_command,
@ -1378,6 +1379,7 @@ WHERE
job_description => $job_description,
job_status => $job_status,
modified_date => $modified_date,
now_time => $now_time,
started_seconds_ago => $started_seconds_ago,
updated_seconds_ago => $updated_seconds_ago,
}});
@ -2671,7 +2673,7 @@ sub insert_or_update_jobs
my $job_picked_up_at = defined $parameter->{job_picked_up_at} ? $parameter->{job_picked_up_at} : 0;
my $job_updated = defined $parameter->{job_updated} ? $parameter->{job_updated} : time;
my $job_name = defined $parameter->{job_name} ? $parameter->{job_name} : "";
my $job_progress = defined $parameter->{job_progress} ? $parameter->{job_progress} : 0;
my $job_progress = defined $parameter->{job_progress} ? $parameter->{job_progress} : "";
my $job_title = defined $parameter->{job_title} ? $parameter->{job_title} : "";
my $job_description = defined $parameter->{job_description} ? $parameter->{job_description} : "";
my $job_status = defined $parameter->{job_status} ? $parameter->{job_status} : "";
@ -5395,8 +5397,13 @@ sub resync_databases
undef $anvil->data->{db_resync}{$uuid}{history}{sql};
# If the merged array has any entries, push them in.
if (@{$merged} > 0)
my $to_write_count = @{$merged};
if ($to_write_count > 0)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0221", variables => {
to_write => $to_write_count,
host_name => $anvil->Get->host_name({debug => 2, host_uuid => $uuid}),
}});
$anvil->Database->write({uuid => $uuid, query => $merged, source => $THIS_FILE, line => __LINE__});
undef $merged;
}
@ -5879,29 +5886,29 @@ ORDER BY
# Record this table's last modified_date for later comparison. We'll also
# record the schema and host column, if found, to save looking the same thing
# up later if we do need a resync.
$anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{last_updated} = $last_updated;
$anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{row_count} = $row_count;
$anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated} = $last_updated;
$anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count} = $row_count;
$anvil->data->{sys}{database}{table}{$table}{schema} = $schema;
$anvil->data->{sys}{database}{table}{$table}{host_column} = $host_column;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::table::${table}::id::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{last_updated},
"sys::database::table::${table}::id::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{row_count},
"sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
"sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count},
"sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
"sys::database::table::${table}::schema" => $anvil->data->{sys}{database}{table}{$table}{schema},
"sys::database::table::${table}::host_column" => $anvil->data->{sys}{database}{table}{$table}{host_column},
}});
if ($anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{row_count} > $anvil->data->{sys}{database}{table}{$table}{row_count})
if ($anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count} > $anvil->data->{sys}{database}{table}{$table}{row_count})
{
$anvil->data->{sys}{database}{table}{$table}{row_count} = $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{row_count};
$anvil->data->{sys}{database}{table}{$table}{row_count} = $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::table::${table}::row_count" => $anvil->data->{sys}{database}{table}{$table}{row_count},
}});
}
if ($anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{last_updated})
if ($anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{last_updated})
{
$anvil->data->{sys}{database}{table}{$table}{last_updated} = $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{last_updated};
$anvil->data->{sys}{database}{table}{$table}{last_updated} = $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
}});
@ -5918,25 +5925,39 @@ ORDER BY
"sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
"sys::database::table::${table}::row_count" => $anvil->data->{sys}{database}{table}{$table}{row_count},
}});
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{sys}{database}{table}{$table}{id}})
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{sys}{database}{table}{$table}{uuid}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::table::${table}::id::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{last_updated},
"sys::database::table::${table}::id::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{row_count},
"sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
"sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count},
}});
if ($anvil->data->{sys}{database}{table}{$table}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{last_updated})
if ($anvil->data->{sys}{database}{table}{$table}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated})
{
# Resync needed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => { uuid => $uuid }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
"sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => {
uuid => $uuid,
host => $anvil->Get->host_name({debug => 2, host_uuid => $uuid}),
}});
# Mark it as behind.
$anvil->Database->_mark_database_as_behind({debug => $debug, uuid => $uuid});
last;
}
elsif ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{id}{$uuid}{row_count})
elsif ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count})
{
# Resync needed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0219", variables => { uuid => $uuid }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::table::${table}::row_count" => $anvil->data->{sys}{database}{table}{$table}{row_count},
"sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count},
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0219", variables => {
uuid => $uuid,
host => $anvil->Get->host_name({debug => 2, host_uuid => $uuid}),
}});
# Mark it as behind.
$anvil->Database->_mark_database_as_behind({debug => $debug, uuid => $uuid});

48
Anvil/Tools/Get.pm
Unescape View File

@ -439,6 +439,54 @@ sub date_and_time
return($return_string);
}
=head2 host_name
This takes a host UUID and returns the host name (as recorded in the C<< hosts >> table). If the entry is not found, an empty string is returned.
Parameters;
=head3 host_uuid (required)
This is the C<< host_uuid >> to translate into a host name.
=cut
sub host_name
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
my $host_name = "";
my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_uuid => $host_uuid }});
my $query = "
SELECT
host_name
FROM
hosts
WHERE
host_uuid = ".$anvil->data->{sys}{database}{use_handle}->quote($host_uuid).";
";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if ($count == 1)
{
# Found it
$host_name = defined $results->[0]->[0] ? $results->[0]->[0] : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }});
return($host_name);
}
=head2 host_uuid
This returns the local host's system UUID (as reported by 'dmidecode'). If the host UUID isn't available, and the program is not running with root priviledges, C<< #!error!# >> is returned.

5
Anvil/Tools/Log.pm
Unescape View File

@ -270,6 +270,11 @@ sub entry
# Log the file and line, if passed.
my $string = "";
if ($anvil->data->{sys}{log_date})
{
# Keep the debug level super high to avoid Get->date_and_time() going into an infinite loop.
$string .= $anvil->Get->date_and_time({debug => 99}).":";
}
if (($source) && ($line))
{
$string .= "$source:$line; ";

14
cgi-bin/striker
Unescape View File

@ -301,6 +301,7 @@ sub process_power
{
# Record the job!
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2,
file => $THIS_FILE,
line => __LINE__,
job_command => $task eq "poweroff" ? "anvil-manage-power --poweroff -y" : "anvil-manage-power --reboot -y",
@ -308,6 +309,7 @@ sub process_power
job_name => "reboot::system",
job_title => $task eq "poweroff" ? "job_0010" : "job_0009",
job_description => $task eq "poweroff" ? "job_0008" : "job_0006",
job_progress => 0,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
@ -316,6 +318,7 @@ sub process_power
$anvil->data->{form}{body} = $anvil->Template->get({file => "striker.html", name => "system_update_recorded", variables => {
title_id => "",
message_id => "",
reload_url => "/cgi-bin/".$THIS_FILE,
title => $task eq "poweroff" ? "#!string!job_0007!#" : "#!string!job_0005!#",
description => $task eq "poweroff" ? "#!string!job_0008!#" : "#!string!job_0006!#",
}});
@ -348,6 +351,7 @@ sub process_update
{
# Record the job!
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2,
file => $THIS_FILE,
line => __LINE__,
job_command => "anvil-update-system",
@ -355,6 +359,7 @@ sub process_update
job_name => "update::system",
job_title => "job_0003",
job_description => "job_0004",
job_progress => 0,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
@ -363,6 +368,7 @@ sub process_update
$anvil->data->{form}{body} = $anvil->Template->get({file => "striker.html", name => "system_update_recorded", variables => {
title_id => "",
message_id => "",
reload_url => "/cgi-bin/".$THIS_FILE,
title => "#!string!striker_0044!#",
description => "#!string!striker_0088!#",
}});
@ -504,6 +510,8 @@ sub process_sync_page
my $peer_table = "";
foreach my $uuid (keys %{$anvil->data->{database}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid => $uuid }});
next if not $anvil->Validate->is_uuid({uuid => $uuid});
next if $uuid eq $host_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid => $uuid }});
@ -771,6 +779,7 @@ sub add_sync_peer
# Store the job
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2,
file => $THIS_FILE,
line => __LINE__,
job_command => $job_command,
@ -778,6 +787,7 @@ sub add_sync_peer
job_name => "striker-peer::add",
job_title => "job_0011",
job_description => "job_0012",
job_progress => 0,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
@ -786,6 +796,7 @@ sub add_sync_peer
$anvil->data->{form}{body} = $anvil->Template->get({file => "striker.html", name => "system_update_recorded", variables => {
title_id => "",
message_id => "",
reload_url => "/cgi-bin/".$THIS_FILE,
title => "#!string!striker_0044!#",
description => $ping ? "#!string!striker_0103!#" : "#!string!striker_0102!#",
}});
@ -851,6 +862,7 @@ sub configure_striker
{
# User has confirmed, update the system!
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2,
file => $THIS_FILE,
line => __LINE__,
job_command => "anvil-configure-striker",
@ -858,6 +870,7 @@ sub configure_striker
job_name => "configure::network",
job_title => "job_0001",
job_description => "job_0002",
job_progress => 0,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
@ -868,6 +881,7 @@ sub configure_striker
$anvil->data->{form}{body} = $anvil->Template->get({file => "striker.html", name => "network_job_recorded", variables => {
title_id => "",
message_id => "",
reload_url => "/cgi-bin/".$THIS_FILE,
title => "#!string!striker_0044!#",
description => "#!string!striker_0045!#",
}});

2
html/skins/alteeve/main.css
Unescape View File

@ -302,7 +302,7 @@ td {
.menu_title {
color: #dbe0e5;
font-size: 2em;
font-size: 1.3em;
padding-top: 30px;
padding-left: 20px;
padding-bottom: 10px;

4
html/skins/alteeve/striker.html
Unescape View File

@ -547,7 +547,7 @@
<span name="#!variable!message_id!#" id="#!variable!message_id!#" class="config_header2">#!variable!description!#</span>
<br />
<hr />
<a href="/" class="button">Reload</a>
<a href="#!variable!reload_url!#" class="button">#!string!striker_0053!#</a>
</td>
</tr>
</div>
@ -563,7 +563,7 @@
<span name="#!variable!message_id!#" id="#!variable!message_id!#" class="config_header2">#!variable!description!#</span>
<br />
<hr />
<a href="/" class="button">Reload</a>
<a href="#!variable!reload_url!#" class="button">#!string!striker_0053!#</a>
</td>
</tr>
</div>

5
share/words.xml
Unescape View File

@ -254,7 +254,7 @@ The database connection error was:
<key name="log_0103">The local machine's UUID was not read properly. It should be stored in: [#!data!sys::host_uuid!#] and contain hexadecimal characters in the format: '012345-6789-abcd-ef01-23456789abcd' and usually matches the output of 'dmidecode --string system-uuid'. If this file exists and if there is a string in the file, please verify that it is structured correctly.</key>
<key name="log_0104">The database with UUID: [#!variable!uuid!#] for: [#!variable!file!#] is behind.</key>
<key name="log_0105">Anvil! database: [#!variable!database!#] already exists.</key>
<key name="log_0106">The database with UUID: [#!variable!uuid!#] is behind. A database resync will be requested.</key>
<key name="log_0106">The database on: [#!variable!host!#] (UUID: [#!variable!uuid!#]) is behind. A database resync will be requested.</key>
<key name="log_0107">[ Warning ] - Failed to delete the temporary postgres password.</key>
<key name="log_0108"><![CDATA[[ Error ] - The method Database->insert_or_update_states() was called but the 'state_host_uuid' parameter was not passed or it is empty. Normally this is set to 'sys::data_uuid'.]]></key>
<key name="log_0109">[ Error ] - Failed to create the Anvil! database: [#!variable!database!#]</key>
@ -367,8 +367,9 @@ The database connection error was:
<key name="log_0216">The #!variable!uuid_name!#: [#!variable!uuid!#] was passed in, but no record with that UUID was found in the database.</key>
<key name="log_0217">The variable with variable_uuid: [#!variable!variable_uuid!#], variable_source_table: [#!variable!variable_source_table!#] and variable_source_uuid: [#!variable!variable_source_uuid!#] was not found in the database, so unable to update.</key>
<key name="log_0218">The variable: [#!variable!name!#] was expected to be an array reference, but it wasn't. It contained (if anything): [#!variable!value!#].</key>
<key name="log_0219">The database with UUID: [#!variable!uuid!#] is missing rows. A database resync will be requested.</key>
<key name="log_0219">The database on: [#!variable!host!#] (UUID: [#!variable!uuid!#]) is missing rows. A database resync will be requested.</key>
<key name="log_0220"><![CDATA[Database->insert_or_update_jobs() was called with 'update_progress_only' but without a 'job_uuid' being set.]]></key>
<key name="log_0221">Writing: [#!variable!to_write!#] record(s) to resync the database on: [#!variable!host_name!#].</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<key name="t_0000">Test</key>

8
tools/anvil-daemon
Unescape View File

@ -356,7 +356,7 @@ sub run_jobs
my $jobs_file = "{\"jobs\":[\n";
# Get a list of pending or incomplete jobs.
my $return = $anvil->Database->get_jobs({ended_within => 300});
my $return = $anvil->Database->get_jobs({debug => 3, ended_within => 300});
my $count = @{$return};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
'return' => $return,
@ -410,7 +410,6 @@ sub run_jobs
if (not exists $anvil->data->{pids}{$job_picked_up_by})
{
# If the job is done, just clear the 'job_picked_up_by' and be done.
$anvil->Job->clear({job_uuid => $job_uuid});
if ($job_progress ne "100")
{
# The previous job is gone, but the job isn't finished. Start it again.
@ -427,6 +426,9 @@ sub run_jobs
job_progress => $job_progress,
job_status => $job_status,
}});
# Clear the job.
$anvil->Job->clear({debug => 3, job_uuid => $job_uuid});
}
# Clear the PID
@ -484,7 +486,7 @@ sub run_jobs
next if $job_progress eq "100";
# If the job is not running, start it.
if ((not $job_picked_up_by) && (not $anvil->data->{switches}{'no-start'}))
if ((not $job_picked_up_by) && ($job_progress ne "100") && (not $anvil->data->{switches}{'no-start'}))
{
# Start the job, appending '--job-uuid' to the command.
my $command = $job_command." --job-uuid ".$job_uuid;

8
tools/anvil-manage-striker-peers
Unescape View File

@ -101,6 +101,8 @@ foreach my $host (sort {$a cmp $b} keys %{$anvil->data->{sorted}{db}})
}});
}
# We're done!
update_progress($anvil, 100, "message_0025");
$anvil->nice_exit({code => 0});
@ -125,7 +127,7 @@ sub update_progress
if (($anvil->data->{jobs}{job_uuid}) && ($anvil->data->{sys}{database}{connections}))
{
$anvil->Job->update_progress({
debug => 2,
debug => 3,
progress => $progress,
message => $message,
job_uuid => $anvil->data->{jobs}{job_uuid},
@ -536,7 +538,6 @@ sub process_entry
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
# Now loop until we see the peer's host_uuid show up
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0212"});
my $peer_host_name = "";
my $host_seen = 0;
until($host_seen)
@ -577,6 +578,7 @@ sub process_entry
# Store the job
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2,
file => $THIS_FILE,
line => __LINE__,
job_host_uuid => $host_uuid,
@ -585,11 +587,11 @@ sub process_entry
job_name => "striker-peer::add",
job_title => "job_0011",
job_description => "job_0012",
job_progress => 0,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
update_progress($anvil, 95, "message_0076");
}
update_progress($anvil, 100, "message_0025");
return(0);
}

Write Preview
Loading…
Cancel
Save