seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

The theme of these commits is improving DB responsiveness.

Browse Source 
* Created Database->_age_out_data() to delete records from the database that are old enough to no longer be useful. This is designed to significantly reduce the size of the database, allowing a better focus on performance.
* Changed Database->connect() to default to NOT check for resync, reworking the old 'no_resync' to 'check_for_resync', so that resync checks happen on demard, instead of by default.
* Updated get_tables_from_schema() to now allow 'schema_file' to be set to 'all', which then loads the schema files of all scan agents as well as the core anvil schema file. Fixed a bug where commented out tables were being counted.
* Re-enabled triggering resyncs on 'last_updated' differences.
* Fixed a bug in scan-ipmitool where the history_id column in history.scan_ipmitool_value was incorrect.
* Created a new tool called striker-show-db-counts that shows the number of records in all public and history schema tables for all databases.
* Updated anvil-update-states to detect when a libvirtd NAT'ed bridge exists and to delete it when found.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 4 years ago
parent 7623b9df87
commit 6abe06f125
12 changed files with 650 additions and 111 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      Anvil/Tools.pm
  2. 510
      Anvil/Tools/Database.pm
  3. 6
      Anvil/Tools/ScanCore.pm
  4. 2
      scancore-agents/scan-ipmitool/scan-ipmitool.sql
  5. 4
      share/words.xml
  6. 1
      tools/Makefile.am
  7. 2
      tools/anvil-daemon
  8. 2
      tools/anvil-join-anvil
  9. 49
      tools/anvil-update-states
  10. 17
      tools/scancore
  11. 3
      tools/striker-manage-install-target
  12. 160
      tools/striker-show-db-counts

5
Anvil/Tools.pm
Unescape View File

@ -839,6 +839,11 @@ sub _set_defaults
# Delay between scans?
run_interval => 30,
},
database => {
# This is the number of hours, after which, transient data (like temperature and
# power data) is considered "old" and gets deleted from the database.
age_out => 48,
},
};
$anvil->data->{sys} = {
apache => {

510
Anvil/Tools/Database.pm
Unescape View File

@ -91,6 +91,7 @@ my $THIS_FILE = "Database.pm";
# resync_databases
# update_host_status
# write
# _age_out_data
# _archive_table
# _find_column
# _find_behind_database
@ -157,6 +158,7 @@ sub parent
# Public methods #
#############################################################################################################
=head2 archive_database
This method takes an array reference of database tables and check each to see if their history schema version needs to be archived or not.
@ -529,29 +531,35 @@ sub check_agent_data
}});
}
# Now check to see if a resync is required, it likely is.
if ($anvil->data->{sys}{database}{connections} > 1)
{
# The source is the agent
$anvil->Database->_find_behind_databases({
debug => $debug,
source => $agent,
tables => $tables,
});
}
# Hold if a lock has been requested.
$anvil->Database->locking({debug => $debug});
# Mark that we're not active.
$anvil->Database->mark_active({debug => $debug, set => 1});
# Sync the database, if needed.
$anvil->Database->resync_databases({debug => $debug});
}
}
# Now check to see if a resync is required...
if ($anvil->data->{sys}{database}{connections} > 1)
{
# The source is the agent
$anvil->Database->_find_behind_databases({
debug => $debug,
source => $agent,
tables => $tables,
});
}
# Hold if a lock has been requested.
$anvil->Database->locking({debug => $debug});
# Mark that we're not active.
# Mark that we're now active.
$anvil->Database->mark_active({debug => $debug, set => 1});
# Sync the database, if needed.
$anvil->Database->resync_databases({debug => $debug});
return(0);
}
@ -1072,6 +1080,14 @@ This module will return the number of databases that were successfully connected
Parameters;
=head3 check_for_resync (optional, default 0)
If set to C<< 1 >>, and there are 2 or more databases available, a check will be make to see if the databases need to be resync'ed or not. This is also set if the command line switch C<< --resync-db >> is used.
B<< Note >>: For daemons like C<< anvil-daemon >> and C<< scancore >>, when a loop starts the current number of available databases is checked against the last number. If the new number is greater, a DB resync check is triggered.
This can be expensive so should not be used in cases where responsiveness is important. It should be used if differences in data could cause issues.
=head3 check_if_configured (optional, default '0')
If set to C<< 1 >>, and if this is a locally hosted database, a check will be made to see if the database is configured. If it isn't, it will be configured.
@ -1086,10 +1102,6 @@ If set, the connection will be made only to the database server matching the UUI
If set to C<< 1 >>, no attempt to ping a target before connection will happen, even if C<< database::<uuid>::ping = 1 >> is set.
=head3 no_resync (optional, default 0)
If set to C<< 1 >>, no checks will be made to resync the database. Generally this is only useful to scan agents (as ScanCore itself is better at detecting and resyncing).
=head3 source (optional)
The C<< source >> parameter is used to check the special C<< updated >> table on all connected databases to see when that source (program name, usually) last updated a given database. If the date stamp is the same on all connected databases, nothing further happens. If one of the databases differ, however, a resync will be requested.
@ -1147,7 +1159,7 @@ sub connect
my $check_if_configured = defined $parameter->{check_if_configured} ? $parameter->{check_if_configured} : 0;
my $db_uuid = defined $parameter->{db_uuid} ? $parameter->{db_uuid} : "";
my $no_ping = defined $parameter->{no_ping} ? $parameter->{no_ping} : 0;
my $no_resync = defined $parameter->{no_resync} ? $parameter->{no_resync} : 0;
my $check_for_resync = defined $parameter->{check_for_resync} ? $parameter->{check_for_resync} : 0;
my $source = defined $parameter->{source} ? $parameter->{source} : "core";
my $sql_file = defined $parameter->{sql_file} ? $parameter->{sql_file} : $anvil->data->{path}{sql}{'anvil.sql'};
my $tables = defined $parameter->{tables} ? $parameter->{tables} : "";
@ -1156,20 +1168,27 @@ sub connect
check_if_configured => $check_if_configured,
db_uuid => $db_uuid,
no_ping => $no_ping,
no_resync => $no_resync,
check_for_resync => $check_for_resync,
source => $source,
sql_file => $sql_file,
tables => $tables,
test_table => $test_table,
}});
# If I wasn't passed an array reference of tables, use the core tables.
# If I wasn't passed an array reference of tables, load them from file(s).
if (not $tables)
{
$tables = $anvil->Database->get_tables_from_schema({debug => $debug, schema_file => $anvil->data->{path}{sql}{'anvil.sql'}});
$tables = $anvil->Database->get_tables_from_schema({debug => $debug, schema_file => "all"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { tables => $tables }});
}
$anvil->data->{switches}{'resync-db'} = "" if not defined $anvil->data->{switches}{'resync-db'};
if ($anvil->data->{switches}{'resync-db'})
{
$check_for_resync = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check_for_resync => $check_for_resync }});
}
my $start_time = [gettimeofday];
#print "Start time: [".$start_time->[0].".".$start_time->[1]."]\n";
@ -1710,43 +1729,35 @@ sub connect
$anvil->Database->disconnect({debug => $debug});
}
# For now, we just find which DBs are behind and let each agent deal with bringing their tables up to
# date.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::connections" => $anvil->data->{sys}{database}{connections},
no_resync => $no_resync,
}});
if (($anvil->data->{sys}{database}{connections} > 1) && (not $no_resync))
# If we have a previous count and the new count is higher, resync.
if (exists $anvil->data->{sys}{database}{last_db_count})
{
# If we have a "last_db_count" and it's the same as the current number of connections, skip
# checking for a resync. This is done because the databases change constantly so tables like
# jobs, which scancore and anvil-daemon constantly change, doesn't trigger a resync when
# records change mid-check.
my $check = 1;
if (exists $anvil->data->{sys}{database}{last_db_count})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::last_db_count" => $anvil->data->{sys}{database}{last_db_count},
"sys::database::connections" => $anvil->data->{sys}{database}{connections},
}});
if ($anvil->data->{sys}{database}{last_db_count} eq $anvil->data->{sys}{database}{connections})
{
$check = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check => $check }});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { check => $check }});
if ($check)
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::last_db_count" => $anvil->data->{sys}{database}{last_db_count},
"sys::database::connections" => $anvil->data->{sys}{database}{connections},
}});
if ($anvil->data->{sys}{database}{connections} > $anvil->data->{sys}{database}{last_db_count})
{
$anvil->Database->_find_behind_databases({
debug => $debug,
source => $source,
tables => $tables,
});
$check_for_resync = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { check_for_resync => $check_for_resync }});
}
}
# If we have a "last_db_count" and it's the lower than the current number of connections, check for a
# resync.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::connections" => $anvil->data->{sys}{database}{connections},
check_for_resync => $check_for_resync,
}});
if (($anvil->data->{sys}{database}{connections} > 1) && ($check_for_resync))
{
$anvil->Database->_find_behind_databases({
debug => $debug,
source => $source,
tables => $tables,
});
}
$anvil->data->{sys}{database}{last_db_count} = $anvil->data->{sys}{database}{connections};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::last_db_count" => $anvil->data->{sys}{database}{last_db_count} }});
@ -1757,7 +1768,14 @@ sub connect
$anvil->Database->mark_active({debug => $debug, set => 1});
# Sync the database, if needed.
$anvil->Database->resync_databases({debug => $debug});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::resync_needed" => $anvil->data->{sys}{database}{resync_needed},
check_for_resync => $check_for_resync,
}});
if (($check_for_resync) && ($anvil->data->{sys}{database}{resync_needed}))
{
$anvil->Database->resync_databases({debug => $debug});
}
# Add ourselves to the database, if needed.
$anvil->Database->insert_or_update_hosts({debug => $debug});
@ -4574,7 +4592,7 @@ Parameters;
=head3 schema_file (required)
This is the full path to a SQL schema file to look for tables in.
This is the full path to a SQL schema file to look for tables in. If set to C<< all >>, then C<< path::sql::anvil.sql >> will be used, as well schema for all scan agents.
=cut
sub get_tables_from_schema
@ -4598,8 +4616,39 @@ sub get_tables_from_schema
return("!!error!!");
}
my $schema = $anvil->Storage->read_file({debug => $debug, file => $schema_file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { schema => $schema }});
my $schema = "";
if ($schema_file eq "all")
{
# We're loading all schema files. Main first
$schema = $anvil->Storage->read_file({debug => $debug, file => $anvil->data->{path}{sql}{'anvil.sql'}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { schema => $schema }});
$anvil->ScanCore->_scan_directory({
debug => $debug,
directory => $anvil->data->{path}{directories}{scan_agents},
});
# Now all agents
foreach my $agent_name (sort {$a cmp $b} keys %{$anvil->data->{scancore}{agent}})
{
my $sql_path = $anvil->data->{scancore}{agent}{$agent_name}.".sql";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
agent_name => $agent_name,
sql_path => $sql_path,
}});
if (not -e $sql_path)
{
next;
}
$schema .= $anvil->Storage->read_file({debug => $debug, file => $sql_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { schema => $schema }});
}
}
else
{
$schema = $anvil->Storage->read_file({debug => $debug, file => $schema_file});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { schema => $schema }});
}
if ($schema eq "!!error!!")
{
@ -4609,7 +4658,7 @@ sub get_tables_from_schema
foreach my $line (split/\n/, $schema)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
$line =~ s/--.*?//;
$line =~ s/--.*$//;
if ($line =~ /CREATE TABLE history\.(.*?) \(/)
{
@ -4644,7 +4693,6 @@ sub get_tables_from_schema
'sys::database::check_tables' => $anvil->data->{sys}{database}{check_tables},
}});
my $table_count = @{$tables};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { table_count => $table_count }});
@ -14134,7 +14182,7 @@ sub manage_anvil_conf
$anvil->Storage->read_config({file => $anvil->data->{path}{configs}{'anvil.conf'}});
# Reconnect
$anvil->Database->connect();
$anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, key => "log_0132"});
}
}
@ -15471,6 +15519,287 @@ sub write
# Private functions #
#############################################################################################################
=head2 _age_out_data
This deletes any data considered transient (power, thermal, etc) after C<< scancore::database::age_out >> hours old.
=cut
sub _age_out_data
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->_age_out_data()" }});
# Get the timestamp to delete jobs and processed alert records older than 2h
my $query = "SELECT now() - '2h'::interval";
my $old_timestamp = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
query => $query,
old_timestamp => $old_timestamp,
}});
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
my $queries = [];
my $query = "SELECT job_uuid FROM jobs WHERE modified_date <= '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $job_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { job_uuid => $job_uuid }});
# Delete
my $query = "DELETE FROM history.jobs WHERE job_uuid = ".$anvil->Database->quote($job_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query;
$query = "DELETE FROM jobs WHERE job_uuid = ".$anvil->Database->quote($job_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query;
}
my $commits = @{$queries};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { commits => $commits }});
if ($commits)
{
# Commit the DELETEs.
$anvil->Database->write({debug => $debug, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__});
}
}
# Remove old processed alerts.
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
my $queries = [];
my $query = "SELECT alert_uuid FROM alerts WHERE alert_processed = 1 AND modified_date <= '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $alert_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { alert_uuid => $alert_uuid }});
# Delete
my $query = "DELETE FROM history.alerts WHERE alert_uuid = ".$anvil->Database->quote($alert_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query;
$query = "DELETE FROM alerts WHERE alert_uuid = ".$anvil->Database->quote($alert_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query;
}
my $commits = @{$queries};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { commits => $commits }});
if ($commits)
{
# Commit the DELETEs.
$anvil->Database->write({debug => $debug, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__});
}
}
# Now process power and tempoerature, if not disabled.
my $age = $anvil->data->{scancore}{database}{age_out};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { age => $age }});
if ($age =~ /\D/)
{
# Age is not valid, set it to defaults.
$age = 48;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { age => $age }});
}
if ($age == 0)
{
# Disabled, return.
return(0);
}
# Get the timestamp to delete thermal and power records older than $age hours.
$query = "SELECT now() - '".$age."h'::interval;";
$old_timestamp = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
query => $query,
old_timestamp => $old_timestamp,
}});
# Purge temperature and power data.
my $tables = {};
$tables->{temperature} = "temperature_uuid";
$tables->{power} = "power_uuid";
$tables->{ip_addresses} = "ip_address_uuid";
foreach my $table (sort {$a cmp $b} keys %{$tables})
{
my $uuid_column = $tables->{$table};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
table => $table,
uuid_column => $uuid_column,
}});
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
my $queries = [];
my $query = "SELECT ".$uuid_column." FROM ".$table;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $column_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { column_uuid => $column_uuid }});
# Find how many records will be left. If it's 0, we'll use an OFFSET 1.
my $query = "SELECT history_id FROM history.".$table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date > '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if ($count)
{
# At least one record will be left.
my $query = "DELETE FROM history.".$table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND modified_date <= '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query;
}
else
{
# This would delete everything, reserve at least one record.
foreach my $row (@{$results})
{
my $history_id = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }});
my $query = "DELETE FROM history.".$table." WHERE ".$uuid_column." = ".$anvil->Database->quote($column_uuid)." AND hostory_id = '".$history_id."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
push @{$queries}, $query;
}
}
}
my $commits = @{$queries};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { commits => $commits }});
if ($commits)
{
# Commit the DELETEs.
$anvil->Database->write({debug => $debug, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__});
}
}
}
### Looks for scan agent data that grows quickly.
# scan-ipmitool
$query = "SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE tablename='scan_ipmitool_values' AND schemaname='public';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
if ($count)
{
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid => $uuid }});
my $queries = [];
my $query = "SELECT scan_ipmitool_value_uuid FROM scan_ipmitool_values;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
my $column_uuid = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { column_uuid => $column_uuid }});
# Find how many records will be left. If it's 0, we'll use an OFFSET 1.
my $query = "SELECT history_id FROM history.scan_ipmitool_values WHERE scan_ipmitool_value_uuid = ".$anvil->Database->quote($column_uuid)." AND modified_date > '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $results = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
if ($count)
{
# At least one record will be left.
my $query = "DELETE FROM history.scan_ipmitool_values WHERE scan_ipmitool_value_uuid = ".$anvil->Database->quote($column_uuid)." AND modified_date <= '".$old_timestamp."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
push @{$queries}, $query;
}
else
{
# This would delete everything, reserve at least one record.
foreach my $row (@{$results})
{
my $history_id = $row->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { history_id => $history_id }});
my $query = "DELETE FROM history.scan_ipmitool_values WHERE scan_ipmitool_value_uuid = ".$anvil->Database->quote($column_uuid)." AND hostory_id = '".$history_id."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
push @{$queries}, $query;
}
}
}
my $commits = @{$queries};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { commits => $commits }});
if ($commits)
{
# Commit the DELETEs.
$anvil->Database->write({debug => $debug, uuid => $uuid, query => $queries, source => $THIS_FILE, line => __LINE__});
}
}
}
# VACCUM
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
my $query = "VACUUM FULL;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Database->write({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
}
return(0);
}
=head2 _archive_table
NOTE: Not implemented yet (will do so once enough records are in the DB.)
@ -15887,6 +16216,7 @@ sub _find_behind_databases
$anvil->data->{sys}{database}{table}{$table}{row_count} = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
"sys::database::table::${table}::row_count" => $anvil->data->{sys}{database}{table}{$table}{row_count},
}});
}
@ -16032,13 +16362,13 @@ ORDER BY
}
else
{
### TODO: Find the table in a .sql file and load it.
### NOTE: We could recover a lost table here if we tried to find the table in
### a .sql file and load it. Might be worth adding later.
}
}
}
# Are being asked to trigger a resync?
$anvil->data->{switches}{'resync-db'} = "" if not defined $anvil->data->{switches}{'resync-db'};
foreach my $uuid (keys %{$anvil->data->{database}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
@ -16077,36 +16407,28 @@ ORDER BY
"sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
"sys::database::table::${table}::uuid::${uuid}::row_count" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count},
}});
### TODO: Use locking to check for resync so things don't change during checks
# if ($anvil->data->{sys}{database}{table}{$table}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated})
# {
# ### NOTE: This triggers often with just a few seconds difference, which is
# ### more likely caused by one database doing reads, something changes,
# ### and the next database is read. As such, we won't trigger unless
# ### the difference is more than 10 seconds.
# # Resync needed.
# my $difference = $anvil->data->{sys}{database}{table}{$table}{last_updated} - $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated};
# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
# "s1:difference" => $anvil->Convert->add_commas({number => $difference }),
# "s2:sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
# "s3:sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
# }});
# if ($difference > 10)
# {
# $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => {
# seconds => $difference,
# table => $table,
# uuid => $uuid,
# host => $anvil->Get->host_name_from_uuid({host_uuid => $uuid}),
# }});
#
# # Mark it as behind.
# $anvil->Database->_mark_database_as_behind({debug => $debug, uuid => $uuid});
# last;
# }
# }
# elsif ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count})
if ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count})
if ($anvil->data->{sys}{database}{table}{$table}{last_updated} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated})
{
# Resync needed.
my $difference = $anvil->data->{sys}{database}{table}{$table}{last_updated} - $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:difference" => $anvil->Convert->add_commas({number => $difference }),
"s2:sys::database::table::${table}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{last_updated},
"s3:sys::database::table::${table}::uuid::${uuid}::last_updated" => $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{last_updated},
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, priority => "alert", key => "log_0106", variables => {
seconds => $difference,
table => $table,
uuid => $uuid,
host => $anvil->Get->host_name_from_uuid({host_uuid => $uuid}),
}});
# Mark it as behind.
$anvil->Database->_mark_database_as_behind({debug => $debug, uuid => $uuid});
last;
}
elsif ($anvil->data->{sys}{database}{table}{$table}{row_count} > $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count})
{
# Resync needed.
my $difference = ($anvil->data->{sys}{database}{table}{$table}{row_count} - $anvil->data->{sys}{database}{table}{$table}{uuid}{$uuid}{row_count});

6
Anvil/Tools/ScanCore.pm
Unescape View File

@ -154,10 +154,7 @@ sub agent_startup
}
# Connect to DBs.
$anvil->Database->connect({
debug => $debug,
no_resync => 0,
});
$anvil->Database->connect({debug => $debug});
$anvil->Log->entry({source => $agent, line => __LINE__, level => $debug, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
@ -197,7 +194,6 @@ sub agent_startup
}
return(0);
}

2
scancore-agents/scan-ipmitool/scan-ipmitool.sql
Unescape View File

@ -88,7 +88,7 @@ CREATE TABLE scan_ipmitool_values (
ALTER TABLE scan_ipmitool_values OWNER TO admin;
CREATE TABLE history.scan_ipmitool_values (
history_id uuid,
history_id bigserial,
scan_ipmitool_value_uuid uuid,
scan_ipmitool_value_host_uuid uuid,
scan_ipmitool_value_scan_ipmitool_uuid uuid,

4
share/words.xml
Unescape View File

@ -718,6 +718,9 @@ sys::manage::firewall = 1
<key name="header_0059">Link Drops</key>
<key name="header_0060"><![CDATA[-=[ Bond Status - #!variable!date!# ] =-]]></key>
<key name="header_0061"><![CDATA[-=[ Ctrl + C to exit ] =-]]></key>
<key name="header_0062">Table</key>
<key name="header_0063">public</key> <!-- SQL schema -->
<key name="header_0064">history</key> <!-- SQL schema -->
<!-- Strings used by jobs -->
<key name="job_0001">Configure Network</key>
@ -2450,6 +2453,7 @@ If you are comfortable that the target has changed for a known reason, you can s
<key name="striker_0284">Cancel</key>
<key name="striker_0285">Close</key>
<key name="striker_0286">This controls if 'anvil-safe-start' is enabled on a node.</key>
<key name="striker_0287">The virtio NAT bridge: [#!variable!bridge!#] exists. Removing it...</key>
<!-- These are generally units and appended to numbers -->
<key name="suffix_0001">#!variable!number!#/sec</key>

1
tools/Makefile.am
Unescape View File

@ -44,6 +44,7 @@ dist_sbin_SCRIPTS = \
striker-prep-database \
striker-purge-target \
striker-scan-network \
striker-show-db-counts \
striker-auto-initialize-all
fencedir = $(FASEXECPREFIX)/sbin

2
tools/anvil-daemon
Unescape View File

@ -91,7 +91,7 @@ $anvil->System->_check_anvil_conf();
# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server.
$anvil->Database->connect({check_if_configured => 1});
$anvil->Database->connect({check_if_configured => 1, check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0132"});
# If I have no databases, sleep for a second and then exit (systemd will restart us).

2
tools/anvil-join-anvil
Unescape View File

@ -1664,7 +1664,7 @@ sub check_local_network
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }});
$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""});
}
# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).
my $wait_until = time + 130;
until ($anvil->data->{sys}{database}{connections})

49
tools/anvil-update-states
Unescape View File

@ -139,6 +139,46 @@ sub update_network
ip => {},
};
# Make sure there are no virsh bridges, removing any found.
my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if (($host_type eq "node") or ($host_type eq "dr"))
{
my $shell_call = $anvil->data->{path}{exe}{virsh}." net-list --all --name";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
foreach my $line (split/\n/, $output)
{
$line =~ s/^\s+//;
$line =~ s/\s+$//;
next if not $line;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "striker_0287", variables => { bridge => $line }});
my $shell_call = $anvil->data->{path}{exe}{virsh}." net-destroy ".$line;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
$shell_call = $anvil->data->{path}{exe}{virsh}." net-undefine ".$line;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
}
}
# Walk through the sysfs files.
local(*DIRECTORY);
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0018", variables => { directory => $directory }});
@ -176,7 +216,7 @@ sub update_network
$duplex =~ s/\n$//;
$operational =~ s/\n$//;
$speed =~ s/\n$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
interface => $interface,
speed => $speed,
mac_address => $mac_address,
@ -204,7 +244,7 @@ sub update_network
$ip_address = defined $anvil->data->{network}{$local_host}{interface}{$interface}{ip} ? $anvil->data->{network}{$local_host}{interface}{$interface}{ip} : "";
$subnet_mask = defined $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask} ? $anvil->data->{network}{$local_host}{interface}{$interface}{subnet_mask} : "";
$type = defined $anvil->data->{network}{$local_host}{interface}{$interface}{type} ? $anvil->data->{network}{$local_host}{interface}{$interface}{type} : "interface";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
ip_address => $ip_address,
subnet_mask => $subnet_mask,
type => $type,
@ -314,7 +354,7 @@ sub update_network
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { bridge_stp_enabled => $bridge_stp_enabled }});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
active_interface => $active_interface,
bond_master => $bond_master,
bond_mode => $bond_mode,
@ -373,7 +413,7 @@ sub update_network
if ($ip_address)
{
$anvil->data->{seen}{ip}{$ip_address} = $interface;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "seen::ip::${ip_address}" => $anvil->data->{seen}{ip}{$ip_address} }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "seen::ip::${ip_address}" => $anvil->data->{seen}{ip}{$ip_address} }});
}
# Store new information we found.
@ -966,6 +1006,7 @@ WHERE
{
# This IP address no longer exists on this host, removing it.
my $ip_address_uuid = $anvil->Database->insert_or_update_ip_addresses({
debug => 2,
file => $THIS_FILE,
line => __LINE__,
'delete' => 1,

17
tools/scancore
Unescape View File

@ -18,6 +18,11 @@
# shutdown.
# - Add a '--silence-alerts --anvil <name>' and '--restore-alerts --anvil <name>' to temporarily
# disable/re-enable alerts. This is to allow for quiet maintenance without stopping scancore itself.
#
# - Disable resync checks by default, and have a resync check happen on scancore startup, anvil-daemon
# startup, and during configuration.
# - Delete records from temperature and power tables that are older than 48 hours, checking periodically in
# scancore on strikers only. Delete jobs records that are 100% complete for 48 hours or more.
#
use strict;
@ -142,8 +147,6 @@ while(1)
next;
}
### TODO: Left off here
### - Agents are timing out?
# Send alerts.
$anvil->Email->send_alerts({debug => 2});
@ -240,7 +243,7 @@ sub wait_for_database
{
my ($anvil) = @_;
$anvil->Database->connect;
$anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
@ -256,7 +259,7 @@ sub wait_for_database
$anvil->_set_paths();
$anvil->_set_defaults();
$anvil->Storage->read_config();
$anvil->Database->connect;
$anvil->Database->connect({check_for_resync => 1});
if ($anvil->data->{sys}{database}{connections})
{
# We're good
@ -366,6 +369,12 @@ sub startup_tasks
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0620", variables => { uptime => $say_uptime }});
}
}
elsif ($host_type eq "striker")
{
# We're a striker, so we're going to check for / remove transient database records on tables
# that always grow (temperature, power, etc) and whose data loses value as it ages.
}
return(0);
}

3
tools/striker-manage-install-target
Unescape View File

@ -1461,6 +1461,8 @@ sub load_packages
}
}
### TODO: Download 'alteeve-release.noarch' directly.
### NOTE: If/when we support other archs, removing '.x86_64/.noarch' would cause all available archs
### to be downloaded (including .ix86, which would waste space...). Decide if it's best to
### explicitely declare archs vs using space/bandwidth to just grab all available.
@ -1475,7 +1477,6 @@ sub load_packages
"adwaita-gtk2-theme.x86_64",
"adwaita-icon-theme.noarch",
"alsa-lib.x86_64",
"alteeve-release.noarch",
"annobin.x86_64",
"anvil-core.noarch",
"anvil-dr.noarch",

160
tools/striker-show-db-counts
Unescape View File

@ -0,0 +1,160 @@
#!/usr/bin/perl
#
# This shows the total number of rows in the public and history (where applicable) schemas of all available
# databases. It is meant as a diagnostic tool
#
use strict;
use warnings;
use Anvil::Tools;
use Data::Dumper;
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
$anvil->Database->connect({debug => 3});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
$anvil->Get->switches();
my $table_length = 0;
my $count_length = 0;
my $db_length = 0;
my $tables = $anvil->Database->get_tables_from_schema({schema_file => "all"});
foreach my $table (@{$tables})
{
if (length($table) > $table_length)
{
$table_length = length($table);
}
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
$anvil->data->{counts}{$table}{$uuid}{public_count} = 0;
$anvil->data->{counts}{$table}{$uuid}{history_count} = 0;
if ($anvil->data->{sys}{database}{history_table}{$table})
{
my $query = "SELECT COUNT(*) FROM history.".$table.";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
$anvil->data->{counts}{$table}{$uuid}{history_count} = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $say_count = $anvil->Convert->add_commas({number => $anvil->data->{counts}{$table}{$uuid}{history_count}});
$anvil->data->{counts}{$table}{$uuid}{history_comma} = $say_count;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"counts::${table}::${uuid}::history_count" => $anvil->data->{counts}{$table}{$uuid}{history_count},
"counts::${table}::${uuid}::history_comma" => $anvil->data->{counts}{$table}{$uuid}{history_comma},
}});
if (length($say_count) > $count_length)
{
$count_length = length($say_count);
}
}
else
{
$anvil->data->{counts}{$table}{$uuid}{history_count} = -1;
$anvil->data->{counts}{$table}{$uuid}{history_comma} = "--";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"counts::${table}::${uuid}::history_count" => $anvil->data->{counts}{$table}{$uuid}{history_count},
"counts::${table}::${uuid}::history_comma" => $anvil->data->{counts}{$table}{$uuid}{history_comma},
}});
}
my $query = "SELECT COUNT(*) FROM ".$table.";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
$anvil->data->{counts}{$table}{$uuid}{public_count} = $anvil->Database->query({uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $say_count = $anvil->Convert->add_commas({number => $anvil->data->{counts}{$table}{$uuid}{public_count}});
$anvil->data->{counts}{$table}{$uuid}{public_comma} = $say_count;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"counts::${table}::${uuid}::public_count" => $anvil->data->{counts}{$table}{$uuid}{public_count},
"counts::${table}::${uuid}::public_comma" => $anvil->data->{counts}{$table}{$uuid}{public_comma},
}});
if (length($say_count) > $count_length)
{
$count_length = length($say_count);
}
}
}
$db_length = (($count_length * 2) + 3);
foreach my $uuid (keys %{$anvil->data->{cache}{database_handle}})
{
my $host_name = $anvil->Get->host_name_from_uuid({host_uuid => $uuid});
$host_name =~ s/\..*$//;
if (length($host_name) > $db_length)
{
$db_length = length($host_name);
}
$anvil->data->{host_uuid}{$uuid}{host_name} = $host_name;
$anvil->data->{db_host_name}{$host_name}{uuid} = $uuid;
}
if ($db_length > (($count_length * 2) - 3))
{
$count_length = (($db_length - 3) / 2);
if ($count_length =~ /\./)
{
$count_length = (int($count_length) + 1);
$db_length++;
}
}
# header, line 1, and build break line
my $break_line = "-"; for (1..$table_length) { $break_line .= "-"; };
print " "; for (1..$table_length) { print " "; };
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{db_host_name}})
{
print " | ".sprintf("%-${db_length}s", $host_name);
$break_line .= "-+-"; for (1..$count_length) { $break_line .= "-"; };
$break_line .= "-+-"; for (1..$count_length) { $break_line .= "-"; };
}
$break_line .= "-";
print " \n";
# header, line 2
my $say_table = $anvil->Words->string({key => "header_0062"});
my $say_public = $anvil->Words->string({key => "header_0063"});
my $say_history = $anvil->Words->string({key => "header_0064"});
my $center_table = $anvil->Words->center_text({string => $say_table, width => $table_length});
my $center_public = $anvil->Words->center_text({string => $say_public, width => $count_length});
my $center_history = $anvil->Words->center_text({string => $say_history, width => $count_length});
print " ".$center_table;
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{db_host_name}})
{
print " | ".$center_public." | ".$center_history;
}
print " \n";
print $break_line."\n";
foreach my $table (sort {$a cmp $b} keys %{$anvil->data->{counts}})
{
print " ".sprintf("%-${table_length}s", $table);
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{db_host_name}})
{
my $uuid = $anvil->data->{db_host_name}{$host_name}{uuid};
my $public_rows = $anvil->data->{counts}{$table}{$uuid}{public_comma};
my $history_rows = $anvil->data->{counts}{$table}{$uuid}{history_comma};
#print " | ".$public_rows." | ".$history_rows;
print " | ".sprintf("%${count_length}s", $public_rows)." | ".sprintf("%${count_length}s", $history_rows);
}
print " \n";
}
print $break_line."\n";
$anvil->nice_exit({exit_code => 0});
Write Preview
Loading…
Cancel
Save