* Started working on getting database archiving working. It's close to done, but needs testing/debugging and other finery.

* Created Database->get_host_from_uuid() that takes a host UUID and returns the host's name.
* Reworked Network->find_matches() to return both the match's IP and subnet.
* Finished getting tools/striker-initialize-host to add all known peers to the target, using IPs on the target's available subnet.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 5 years ago
parent b1a175baa0
commit b436500a54
  1. 369
      Anvil/Tools/Database.pm
  2. 23
      Anvil/Tools/Network.pm
  3. 2
      anvil.conf
  4. 3
      share/words.xml
  5. 85
      tools/anvil-update-states
  6. 96
      tools/striker-initialize-host

@ -21,6 +21,7 @@ my $THIS_FILE = "Database.pm";
# connect
# disconnect
# get_alert_recipients
# get_host_from_uuid
# get_hosts
# get_job_details
# get_jobs
@ -138,22 +139,29 @@ sub archive_database
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->archive_database()" }});
my $tables = defined $parameter->{tables} ? $parameter->{tables} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
tables => $tables,
}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { tables => $tables }});
# If the 'tables' parameter is an array reference, add it to 'sys::database::check_tables' (creating
# it, if needed).
if (ref($tables) ne "ARRAY")
# If not given tables, use the system tables.
if (not $tables)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0432"});
$tables = $anvil->data->{sys}{database}{check_tables};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { tables => $tables }});
}
# If this isn't a dashboard, exit.
my $host_type = $anvil->System->get_host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_type => $host_type }});
if ($host_type ne "dashboard")
{
# ...
return(1);
}
# Is archiving disabled?
if (not $anvil->data->{sys}{database}{archive}{trigger})
# If the 'tables' parameter is an array reference, add it to 'sys::database::check_tables' (creating
# it, if needed).
if (ref($tables) ne "ARRAY")
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0189"});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0432"});
return(1);
}
@ -165,15 +173,99 @@ sub archive_database
return(1);
}
# Make sure I have sane values.
$anvil->data->{sys}{database}{archive}{compress} = 1 if not defined $anvil->data->{sys}{database}{archive}{compress};
$anvil->data->{sys}{database}{archive}{count} = 10000 if not defined $anvil->data->{sys}{database}{archive}{count};
$anvil->data->{sys}{database}{archive}{division} = 25000 if not defined $anvil->data->{sys}{database}{archive}{division};
$anvil->data->{sys}{database}{archive}{trigger} = 20000 if not defined $anvil->data->{sys}{database}{archive}{trigger};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::archive::compress" => $anvil->data->{sys}{database}{archive}{compress},
"sys::database::archive::count" => $anvil->data->{sys}{database}{archive}{count},
"sys::database::archive::division" => $anvil->data->{sys}{database}{archive}{division},
"sys::database::archive::trigger" => $anvil->data->{sys}{database}{archive}{trigger},
}});
# Make sure the archive directory is sane.
if ((not defined $anvil->data->{sys}{database}{archive}{directory}) or ($anvil->data->{sys}{database}{archive}{directory} !~ /^\//))
{
$anvil->data->{sys}{database}{archive}{directory} = "/usr/local/anvil/archives/";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::archive::directory" => $anvil->data->{sys}{database}{archive}{directory},
}});
}
if (not -d $anvil->data->{sys}{database}{archive}{directory})
{
my $failed = $anvil->Storage->make_directory({
debug => $debug,
directory => $anvil->data->{sys}{database}{archive}{directory},
mode => "0700",
user => "root",
group => "root",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { failed => $failed }});
if ($failed)
{
# ...
return("!!error!!");
}
}
# Make sure the numerical values are sane
if ($anvil->data->{sys}{database}{archive}{count} !~ /^\d+$/)
{
# Use the set value if it just has commas.
$anvil->data->{sys}{database}{archive}{count} =~ s/,//g;
$anvil->data->{sys}{database}{archive}{count} =~ s/\.\d+$//g;
if ($anvil->data->{sys}{database}{archive}{count} !~ /^\d+$/)
{
$anvil->data->{sys}{database}{archive}{count} = 10000;
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::archive::count" => $anvil->data->{sys}{database}{archive}{count},
}});
}
if ($anvil->data->{sys}{database}{archive}{division} !~ /^\d+$/)
{
# Use the set value if it just has commas.
$anvil->data->{sys}{database}{archive}{division} =~ s/,//g;
$anvil->data->{sys}{database}{archive}{division} =~ s/\.\d+$//g;
if ($anvil->data->{sys}{database}{archive}{division} !~ /^\d+$/)
{
$anvil->data->{sys}{database}{archive}{division} = 25000;
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::archive::division" => $anvil->data->{sys}{database}{archive}{division},
}});
}
if ($anvil->data->{sys}{database}{archive}{trigger} !~ /^\d+$/)
{
# Use the set value if it just has commas.
$anvil->data->{sys}{database}{archive}{trigger} =~ s/,//g;
$anvil->data->{sys}{database}{archive}{trigger} =~ s/\.\d+$//g;
if ($anvil->data->{sys}{database}{archive}{trigger} !~ /^\d+$/)
{
$anvil->data->{sys}{database}{archive}{trigger} = 20000;
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::database::archive::trigger" => $anvil->data->{sys}{database}{archive}{trigger},
}});
}
# Is archiving disabled?
if (not $anvil->data->{sys}{database}{archive}{trigger})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0189"});
return(1);
}
# We'll use the list of tables created for _find_behind_databases()'s 'sys::database::check_tables'
# array, but in reverse so that tables with primary keys (first in the array) are archived last.
foreach my $table (reverse(@{$tables}))
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { table => $table }});
$anvil->Database->_archive_table({table => $table});
$anvil->Database->_archive_table({debug => $debug, table => $table});
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0126", variables => { method => "Database->archive_database()" }});
return(0);
}
@ -1211,9 +1303,6 @@ sub connect
# Mark that we're not active.
$anvil->Database->mark_active({debug => $debug, set => 1});
# Archive old data.
$anvil->Database->archive_database({debug => $debug});
# Sync the database, if needed.
$anvil->Database->resync_databases({debug => $debug});
@ -1309,6 +1398,68 @@ WHERE
return();
}
=head2 get_host_from_uuid
This takes a host UUID and returns the host's name. If there is a problem, or if the host UUID isn't found, an empty string is returned.
Parameters;
=head3 host_uuid (required)
This is the host UUID we're querying the name of.
=head3 short (optional, default '0')
If set to C<< 1 >>, the short host name is returned. When set to C<< 0 >>, the full host name is returned.
=cut
sub get_host_from_uuid
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->get_host_from_uuid()" }});
my $host_name = "";
my $host_uuid = defined $parameter->{host_uuid} ? $parameter->{host_uuid} : "";
my $short = defined $parameter->{short} ? $parameter->{short} : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
host_uuid => $host_uuid,
short => $short,
}});
if (not $host_uuid)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->get_host_from_uuid()", parameter => "host_uuid" }});
return($host_name);
}
my $query = "SELECT host_name FROM hosts WHERE host_uuid = ".$anvil->Database->quote($host_uuid).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
if ($count)
{
$host_name = $results->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }});
if ($short)
{
$host_name =~ s/\..*$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { host_name => $host_name }});
}
}
return($host_name);
}
=head2 get_hosts
Get a list of hosts from the c<< hosts >> table, returned as an array of hash references.
@ -1412,7 +1563,7 @@ sub get_job_details
if (not $job_uuid)
{
# Throw an error and exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->get_job_details()", parameter => "get_jobs" }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0020", variables => { method => "Database->get_job_details()", parameter => "job_uuid" }});
return($return);
}
@ -4357,8 +4508,9 @@ sub insert_or_update_network_interfaces
# If we don't have a network interface UUID, try to look one up using the MAC address
if (not $network_interface_uuid)
{
# See if I know this NIC by referencing it's MAC.
my $query = "SELECT network_interface_uuid FROM network_interfaces WHERE network_interface_mac_address = ".$anvil->Database->quote($network_interface_mac_address).";";
# See if I know this NIC by referencing it's MAC and name. The name is needed because virtual
# devices can share the MAC with the real interface.
my $query = "SELECT network_interface_uuid FROM network_interfaces WHERE network_interface_mac_address = ".$anvil->Database->quote($network_interface_mac_address)." AND network_interface_name = ".$anvil->Database->quote($network_interface_name).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$network_interface_uuid = $anvil->Database->query({query => $query, source => $file ? $file." -> ".$THIS_FILE : $THIS_FILE, line => $line ? $line." -> ".__LINE__ : __LINE__})->[0]->[0];
@ -6949,7 +7101,7 @@ Example;
B<< NOTE >>:
Unlike most Anvil methods, this one does NOT use hashes for the parameters! It is meant to replicate C<< DBI->quite("foo") >>, so the only passed-in value is the string to quote. If an undefined or empty string is passed in, a quoted empty string will be returned.
Unlike most Anvil methods, this one does NOT use hashes for the parameters! It is meant to replicate C<< DBI->quote("foo") >>, so the only passed-in value is the string to quote. If an undefined or empty string is passed in, a quoted empty string will be returned.
=cut
sub quote
@ -7149,7 +7301,7 @@ sub resync_databases
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 2;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->resync_databases()" }});
# If a resync isn't needed, just return.
@ -7161,6 +7313,10 @@ sub resync_databases
return(0);
}
# Archive old data before resync'ing
$anvil->Database->archive_database({debug => $debug});
die;
### NOTE: Don't sort this array, we need to resync in the order that the user passed the tables to us
### to avoid trouble with primary/foreign keys.
# We're going to use the array of tables assembles by _find_behind_databases() stored in
@ -7808,16 +7964,8 @@ sub _archive_table
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Database->_archive_table()" }});
my $table = $parameter->{table} ? $parameter->{table} : "";
my $offset = $parameter->{offset} ? $parameter->{offset} : 0;
my $loop = $parameter->{loop} ? $parameter->{loop} : 0;
my $division = $parameter->{division} ? $parameter->{division} : $anvil->data->{sys}{database}{archive}{division};
my $compress = $parameter->{compress} ? $parameter->{compress} : $anvil->data->{sys}{database}{archive}{compress};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
table => $table,
offset => $offset,
loop => $loop,
division => $division,
compress => $compress,
}});
if (not $table)
@ -7826,48 +7974,177 @@ sub _archive_table
return("!!error!!");
}
# These values are sanity checked before this method is called.
my $compress = $anvil->data->{sys}{database}{archive}{compress};
my $directory = $anvil->data->{sys}{database}{archive}{directory};
my $drop_to = $anvil->data->{sys}{database}{archive}{count};
my $division = $anvil->data->{sys}{database}{archive}{division};
my $trigger = $anvil->data->{sys}{database}{archive}{trigger};
my $time_stamp = $anvil->Get->date_and_time({file_name => 1});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
compress => $compress,
directory => $directory,
drop_to => $drop_to,
division => $division,
trigger => $trigger,
time_stamp => $time_stamp,
}});
# Loop through each database so that we archive from everywhere before resync'ing.
foreach my $uuid (keys %{$anvil->data->{database}})
{
# First, if this table doesn't have a history schema, exit.
my $vacuum = 0;
my $query = "SELECT COUNT(*) FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = 'history' AND table_name = ".$anvil->Database->quote($table).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
my $count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
my $count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { count => $count }});
if (not $count)
{
# History table doesn't exist, we're done.
return(0);
next;
}
# Before we do any real analysis, do we have enough entries in the history schema to trigger an archive?
$query = "SELECT COUNT(*) FROM history.".$table.";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$count = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:count" => $count,
"s2:sys::database::archive::trigger" => $anvil->data->{sys}{database}{archive}{trigger},
$count = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:uuid" => $uuid,
"s2:count" => $count,
}});
if ($count <= $anvil->data->{sys}{database}{archive}{trigger})
if ($count <= $trigger)
{
# History table doesn't exist, we're done.
return(0);
# Not enough records to bother archiving.
next;
}
# There is enough data to trigger an archive, so lets get started with a list of columns in this
# table.
# If there are more than
my $to_remove = $count - $drop_to;
my $loops = (int($to_remove / $division) + 1);
my $records_per_loop = $anvil->Convert->round({number => ($to_remove / $loops)});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:to_remove" => $to_remove,
"s2:loops" => $loops,
"s3:records_per_loop" => $records_per_loop,
}});
# There is enough data to trigger an archive, so lets get started with a list of columns in
# this table.
$query = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'history' AND table_name = ".$anvil->Database->quote($table)." AND column_name != 'history_id' AND column_name != 'modified_date';";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0124", variables => { query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $columns = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $columns = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $column_count = @{$columns};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
columns => $columns,
column_count => $column_count
}});
#print Dumper $columns;
my $offset = $count - $records_per_loop;
my $loop = 0;
for (1..$loops)
{
# We need to date stamp from the closest record to the offset.
$loop++;
my $sql_file = "COPY ".$table." (";
my $query = "SELECT modified_date FROM history.".$table." OFFSET ".$offset." LIMIT 1";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:loop" => $loop,
"s2:query" => $query,
"s3:sql_file" => $sql_file,
}});
# See m2's DB->archive_if_needed() for old version of this.
my $modified_date = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { modified_date => $modified_date }});
# Build the query.
$query = "SELECT ";
foreach my $column (sort {$a cmp $b} @{$columns})
{
$sql_file .= $column->[0].", ";
$query .= $column->[0].", ";
}
$sql_file .= "modified_date) FROM stdin;\n";
$query .= "modified_date FROM history.".$table." WHERE modified_date >= '".$modified_date."' ORDER BY modified_date ASC OFFSET ".$offset.";";
my $results = $anvil->Database->query({debug => $debug, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
results => $results,
count => $count,
}});
foreach my $row (@{$results})
{
# Build the string.
my $line = "";
my $i = 0;
foreach my $column (@{$columns})
{
my $value = defined $row->[$i] ? $row->[$i] : '\N';
$i++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:i" => $i,
"s2:column" => $column,
"s3:value" => $value,
}});
# We need to convert tabs and newlines into \t and \n
$value =~ s/\t/\\t/g;
$value =~ s/\n/\\n/g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { value => $value }});
$line .= $value."\t";
}
$sql_file .= $line."\n";
# The 'history_id' is NOT consistent between databases! So we don't record it here.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
}
$sql_file .= "\\.\n\n";;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { sql_file => $sql_file }});
my $archive_file = $directory."/".$anvil->Database->get_host_from_uuid({short => 1, host_uuid => $uuid}).".".$table.".".$time_stamp.".".$loop.".out";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { archive_file => $archive_file }});
# It may not be secure, but we play it safe.
my ($failed) = $anvil->Storage->write_file({
debug => $debug,
body => $sql_file,
file => $archive_file,
user => "root",
group => "root",
mode => "0600",
secure => 1.
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
if ($failed)
{
# ???
last;
}
else
{
$vacuum = 1;
$query = "DELETE FROM history.".$table." WHERE modified_date >= '".$modified_date."';";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
}
$offset -= $records_per_loop;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { offset => $offset }});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { vacuum => $vacuum }});
if ($vacuum)
{
my $query = "VACUUM FULL;";
$anvil->Database->write({debug => 2, uuid => $uuid, query => $query, source => $THIS_FILE, line => __LINE__});
}
die;
}
return(0);
}
@ -7941,7 +8218,7 @@ sub _find_behind_databases
# Look at all the databases and find the most recent time stamp (and the ID of the DB).
my $source_updated_time = 0;
foreach my $uuid (sort {$a cmp $b} keys %{$anvil->data->{database}})
foreach my $uuid (keys %{$anvil->data->{database}})
{
my $database_name = defined $anvil->data->{database}{$uuid}{name} ? $anvil->data->{database}{$uuid}{name} : "#!string!log_0185!#";
my $database_user = defined $anvil->data->{database}{$uuid}{user} ? $anvil->data->{database}{$uuid}{user} : "#!string!log_0185!#";

@ -609,7 +609,16 @@ sub download
=head2 find_matches
This takes two hash keys from prior C<< Network->get_ips() >> runs and finds which are on the same network.
This takes two hash keys from prior C<< Network->get_ips() >> or C<< ->load_ips() >> runs and finds which are on the same network.
A hash reference is returned using the format:
* <first>::<interface>::ip = <ip_address>
* <first>::<interface>::subnet = <subnet_mask>
* <second>::<interface>::ip = <ip_address>
* <second>::<interface>::subnet = <subnet_mask>
Where C<< first >> and C<< second >> are the parameters passed in below and C<< interface >> is the name of the interface on the fist/second machine that can talk to one another.
Paramters;
@ -697,11 +706,15 @@ sub find_matches
if ($first_network eq $second_network)
{
# Match!
$match->{$first}{$first_interface} = $second_network;
$match->{$second}{$second_interface} = $second_network;
$match->{$first}{$first_interface}{ip} = $first_ip;
$match->{$first}{$first_interface}{subnet} = $second_network;
$match->{$second}{$second_interface}{ip} = $second_ip;
$match->{$second}{$second_interface}{subnet} = $first_network;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"${first}::${first_interface}" => $match->{$first}{$first_interface},
"${second}::${second_interface}" => $match->{$second}{$second_interface},
"${first}::${first_interface}::ip" => $match->{$first}{$first_interface}{ip},
"${first}::${first_interface}::subnet" => $match->{$first}{$first_interface}{subnet},
"${second}::${second_interface}::ip" => $match->{$second}{$second_interface}{ip},
"${second}::${second_interface}::subnet" => $match->{$second}{$second_interface}{subnet},
}});
}
}

@ -67,7 +67,7 @@
sys::database::archive::compress = 1
sys::database::archive::count = 10000
sys::database::archive::directory = /usr/local/anvil/archives/
sys::database::archive::division = 25000
sys::database::archive::division = 250000
sys::database::archive::trigger = 20000
# This puts a limit on how many queries (writes, generally) to make in a single batch transaction. This is

@ -1061,6 +1061,9 @@ Failure! The return code: [#!variable!return_code!#] was received ('0' was expec
<key name="job_0065">Refresh the 'OUI' database used to cross reference MAC addresses to the companies that own them.</key>
<key name="job_0066">Network Scan.</key>
<key name="job_0067">This job does a simple ping scan of the networks connected to this host. Any detected hosts have their MAC / IP addresses recorded. This is designed to help determine IP addresses assigned to servers hosted on the Anvil! system.</key>
<key name="job_0068">Adding the database connection information for the dashboard: [#!variable!host_name!#] to the target's anvil.conf file.</key>
<key name="job_0069">Unable to find a matching network, skipping this database.</key>
<key name="job_0070">Something went wrong adding this database. Please see: [#!data!path::log::file!#] for details.</key>
<!-- Warnings -->
<key name="striker_warning_0001">The IP address will change. You will need to reconnect after applying these changes.</key>

@ -20,10 +20,11 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
my $anvil = Anvil::Tools->new();
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 0});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
$anvil->Database->connect({debug => 3});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
die;
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
@ -52,7 +53,7 @@ sub update_network
# * 'network::local::interface::<iface_name>::ip' - If an IP address is set
# * 'network::local::interface::<iface_name>::subnet' - If an IP is set
my $directory = "/sys/class/net";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { directory => $directory }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { directory => $directory }});
# We'll need to know what interfaces to remove, if any. This will store the interfaces we've seen and
# any others will be removed.
@ -64,7 +65,7 @@ sub update_network
# Walk through the sysfs files.
local(*DIRECTORY);
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0018", variables => { directory => $directory }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0018", variables => { directory => $directory }});
opendir(DIRECTORY, $directory);
while(my $file = readdir(DIRECTORY))
{
@ -73,7 +74,7 @@ sub update_network
next if $file eq "lo";
next if $file =~ /virbr\d/;
my $full_path = "$directory/$file";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { full_path => $full_path }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { full_path => $full_path }});
if (-d $full_path)
{
# Pull out the data I want. Note that some of these don't exist with virtio-net interfaces.
@ -94,7 +95,7 @@ sub update_network
$duplex =~ s/\n$//;
$operational =~ s/\n$//;
$speed =~ s/\n$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
interface => $interface,
speed => $speed,
mac_address => $mac_address,
@ -120,7 +121,7 @@ sub update_network
{
$ip_address = $anvil->data->{network}{'local'}{interface}{$interface}{ip} ? $anvil->data->{network}{'local'}{interface}{$interface}{ip} : "";
$subnet_mask = $anvil->data->{network}{'local'}{interface}{$interface}{subnet} ? $anvil->data->{network}{'local'}{interface}{$interface}{subnet} : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
ip_address => $ip_address,
subnet_mask => $subnet_mask,
}});
@ -134,7 +135,7 @@ sub update_network
# It's a slave.
$mac_address = $anvil->Storage->read_file({file => $mac_bond_file});
$mac_address =~ s/\n$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { mac_address => $mac_address }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { mac_address => $mac_address }});
}
# If this is a virtual interface, set some fake values that don't actually exist on
@ -145,7 +146,7 @@ sub update_network
# Speed is "as fast as possible", so we'll record 100 Gbps, but that is really kind of arbitrary.
$speed = 100000 if ((not $speed) or ($speed eq "-1"));
$duplex = "full" if not $duplex;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
speed => $speed,
duplex => $duplex,
}});
@ -154,7 +155,7 @@ sub update_network
if (not $link_state)
{
$speed = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { speed => $speed }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { speed => $speed }});
}
# Is this a bond interface?
@ -181,7 +182,7 @@ sub update_network
$mii_polling_interval =~ s/\n$//;
$up_delay =~ s/\n$//;
$down_delay =~ s/\n$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
active_slave => $active_slave,
bond_mode => $bond_mode,
mii_polling_interval => $mii_polling_interval,
@ -208,7 +209,7 @@ sub update_network
$bridge_stp_enabled = $anvil->Storage->read_file({debug => 3, file => $full_path."/bridge/stp_state"});
$bridge_id =~ s/\n$//;
$bridge_stp_enabled =~ s/\n$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
bridge_id => $bridge_id,
bridge_stp_enabled => $bridge_stp_enabled,
type => $type,
@ -225,10 +226,10 @@ sub update_network
{
$bridge_stp_enabled = "enabled_userland";
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { bridge_stp_enabled => $bridge_stp_enabled }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridge_stp_enabled => $bridge_stp_enabled }});
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
active_slave => $active_slave,
bond_master => $bond_master,
bond_mode => $bond_mode,
@ -263,14 +264,14 @@ sub update_network
# NOTE: This is probably 0 now... Though someday >100 Gbps will be reasonable
# and we'll need to change this.
$speed = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { speed => $speed }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { speed => $speed }});
}
# Find the media, if possible.
my ($ethtool, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{ethtool}." $interface"});
foreach my $line (split/\n/, $ethtool)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { line => $line }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /Supported ports: \[ (.*?) \]/i)
{
$media = lc($1);
@ -306,7 +307,7 @@ sub update_network
type => $type,
up_delay => $up_delay,
};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"network::interfaces::by_name::${interface}::active_slave" => $anvil->data->{network}{interfaces}{by_name}{$interface}{active_slave},
"network::interfaces::by_name::${interface}::bond_mode" => $anvil->data->{network}{interfaces}{by_name}{$interface}{bond_mode},
"network::interfaces::by_name::${interface}::bond_master" => $anvil->data->{network}{interfaces}{by_name}{$interface}{bond_master},
@ -369,7 +370,7 @@ sub update_network
my $default_gateway = $anvil->data->{network}{'local'}{interface}{$interface}{default_gateway};
my $gateway = $anvil->data->{network}{'local'}{interface}{$interface}{gateway};
my $dns = $anvil->data->{network}{'local'}{interface}{$interface}{dns};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
active_slave => $active_slave,
bond_mode => $bond_mode,
bond_master => $bond_master,
@ -537,10 +538,10 @@ WHERE
AND
bond_mode != 'DELETED'
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
@ -561,7 +562,7 @@ AND
bond_mac_address => $row->[10],
bond_operational => $row->[11],
};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"bonds::${bond_uuid}::bond_name" => $anvil->data->{bonds}{$bond_uuid}{bond_name},
"bonds::${bond_uuid}::bond_mode" => $anvil->data->{bonds}{$bond_uuid}{bond_mode},
"bonds::${bond_uuid}::bond_mtu" => $anvil->data->{bonds}{$bond_uuid}{bond_mtu},
@ -602,10 +603,10 @@ WHERE
AND
bridge_id != 'DELETED'
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
$results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
@ -624,7 +625,7 @@ AND
bridge_mtu => $bridge_mtu,
bridge_stp_enabled => $bridge_stp_enabled,
};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"bridges::${bridge_uuid}::bridge_name" => $anvil->data->{bridges}{$bridge_uuid}{bridge_name},
"bridges::${bridge_uuid}::bridge_id" => $anvil->data->{bridges}{$bridge_uuid}{bridge_id},
"bridges::${bridge_uuid}::bridge_mac" => $anvil->data->{bridges}{$bridge_uuid}{bridge_mac},
@ -666,10 +667,10 @@ AND
ORDER BY
modified_date DESC
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
$results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
@ -695,7 +696,7 @@ ORDER BY
network_interface_bond_uuid => defined $row->[9] ? $row->[9] : 'NULL',
network_interface_bridge_uuid => defined $row->[10] ? $row->[10] : 'NULL',
};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"network_interfaces::${network_interface_uuid}::network_interface_mac_address" => $anvil->data->{network_interfaces}{$network_interface_uuid}{network_interface_mac_address},
"network_interfaces::${network_interface_uuid}::network_interface_name" => $anvil->data->{network_interfaces}{$network_interface_uuid}{network_interface_name},
"network_interfaces::${network_interface_uuid}::network_interface_speed" => $anvil->data->{network_interfaces}{$network_interface_uuid}{network_interface_speed},
@ -763,10 +764,10 @@ WHERE
AND
ip_address_on_type != 'DELETED'
;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
$results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
$count = @{$results};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
results => $results,
count => $count,
}});
@ -780,7 +781,7 @@ WHERE
my $ip_address_gateway = $row->[5];
my $ip_address_default_gateway = $row->[6];
my $ip_address_dns = $row->[7];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
ip_address_on_type => $ip_address_on_type,
ip_address_on_uuid => $ip_address_on_uuid,
ip_address_address => $ip_address_address,
@ -794,7 +795,7 @@ WHERE
my $found = 0;
foreach my $interface (sort {$a cmp $b} keys %{$anvil->data->{network}{'local'}{interface}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
interface => $interface,
ip_address_address => $ip_address_address,
"network::local::interface::${interface}::ip" => $anvil->data->{network}{'local'}{interface}{$interface}{ip},
@ -802,29 +803,29 @@ WHERE
if ((defined $anvil->data->{network}{'local'}{interface}{$interface}{ip}) && ($anvil->data->{network}{'local'}{interface}{$interface}{ip} eq $ip_address_address))
{
$found = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { found => $found }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
last;
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { found => $found }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }});
if ($found)
{
my $say_on = "";
if ($ip_address_on_type eq "interface")
{
$say_on = $anvil->data->{network_interfaces}{$ip_address_on_uuid}{network_interface_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_on => $say_on }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_on => $say_on }});
}
elsif ($ip_address_on_type eq "bond")
{
$say_on = $anvil->data->{bonds}{$ip_address_on_uuid}{bond_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_on => $say_on }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_on => $say_on }});
}
elsif ($ip_address_on_type eq "bridge")
{
$say_on = $anvil->data->{bridges}{$ip_address_on_uuid}{bridge_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { say_on => $say_on }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_on => $say_on }});
}
$network_json .= " { \"address\":\"$ip_address_address\", \"on\":\"$say_on\", \"subnet\":\"$ip_address_subnet_mask\", \"gateway\":\"$ip_address_gateway\", \"default_gateway\":\"$ip_address_default_gateway\", \"dns\":\"$ip_address_dns\" },\n";
@ -869,18 +870,16 @@ WHERE
}
}
die;
$network_json =~ s/,$//s;
$network_json .= "]}\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { network_json => $network_json }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_json => $network_json }});
$network_xml .= "</network>\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { network_xml => $network_xml }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_xml => $network_xml }});
# Write the JSON file.
my $output_json = $anvil->data->{path}{directories}{html}."/status/network.json";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output_xml => $output_json }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output_xml => $output_json }});
$anvil->Storage->write_file({
file => $output_json,
body => $network_json,
@ -892,7 +891,7 @@ WHERE
# Write the XML file.
my $output_xml = $anvil->data->{path}{directories}{html}."/status/network.xml";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { output_xml => $output_xml }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output_xml => $output_xml }});
$anvil->Storage->write_file({
file => $output_xml,
body => $network_xml,

@ -152,6 +152,7 @@ sub add_databases
# We'll sort in reverse order, so if there is a BCN address, we'll use it.
my $host_uuid = $anvil->data->{sys}{host_uuid};
my $db_host = "";
my $db_ping = 1,
my $db_port = "";
my $db_password = "";
foreach my $interface (sort {$a cmp $b} keys %{$match->{'local'}})
@ -186,6 +187,7 @@ sub add_databases
port => $anvil->data->{data}{ssh_port},
db_host_uuid => $anvil->data->{sys}{host_uuid},
db_host => $db_host,
db_ping => $db_ping,
db_port => $db_port,
db_password => $db_password,
});
@ -214,8 +216,19 @@ sub add_databases
# Now add any peers we know about.
foreach my $uuid (keys %{$anvil->data->{database}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:uuid' => $uuid,
's2:sys::host_uuid' => $anvil->data->{sys}{host_uuid},
}});
next if $uuid eq $anvil->data->{sys}{host_uuid};
my $target_host = $anvil->Database->get_host_from_uuid({host_uuid => $uuid, short => 1});
$target_host = $target if not $target_host;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_host => $target_host }});
$anvil->data->{job}{progress} += 2;
update_progress($anvil, $anvil->data->{job}{progress}, "job_0068,!!host_name!".$target_host."!!");
# This is a peer.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"database::${uuid}::host" => $anvil->data->{database}{$uuid}{host},
@ -224,9 +237,88 @@ sub add_databases
"database::${uuid}::ping" => $anvil->data->{database}{$uuid}{ping},
}});
# The host may not be able to reach the 'host' that we can. So get the IPs we know about for
# this host and see if we can find a match.
# The host being initialized may not be able to reach the 'host' as set in our config. So get the IPs
# we know about for this host and see if we can find a match.
my $peer_host = "";
my $peer_port = $anvil->data->{database}{$uuid}{port};
my $peer_ping = $anvil->data->{database}{$uuid}{ping};
my $peer_password = $anvil->data->{database}{$uuid}{password};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
peer_host => $peer_host,
peer_port => $peer_port,
peer_password => $anvil->Log->is_secure($peer_password),
peer_ping => $peer_ping,
}});
# Get the IP(s) for the Striker peer.
$anvil->Network->load_ips({host_uuid => $uuid});
# Find a match between the target and the peer.
my ($match) = $anvil->Network->find_matches({
first => $target,
second => $uuid,
});
# Did we find a match?
if ($match)
{
# Yup!
my $match_found = 0;
foreach my $interface (sort {$a cmp $b} keys %{$match->{$uuid}})
{
# Get the IP
$peer_host = $match->{$uuid}{$interface}{ip};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
interface => $interface,
peer_host => $peer_host,
}});
if ($peer_host)
{
my $failed = $anvil->Database->manage_anvil_conf({
debug => 2,
target => $target,
remote_user => "root",
password => $anvil->data->{data}{password},
port => $anvil->data->{data}{ssh_port},
db_host_uuid => $uuid,
db_host => $peer_host,
db_ping => $peer_ping,
db_port => $peer_port,
db_password => $peer_password,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
$anvil->data->{job}{progress} += 3;
if ($failed)
{
# Something went wrong
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, key => "error_0076"});
update_progress($anvil, $anvil->data->{job}{progress}, "job_0070");
}
else
{
# Success! We're done@
$match_found = 1;
update_progress($anvil, $anvil->data->{job}{progress}, "job_0034");
}
last;
}
}
if (not $match_found)
{
# No (workable) match, not sure how this could happen though... Bug in
# 'Network->find_matches()'?
$anvil->data->{job}{progress} += 3;
update_progress($anvil, $anvil->data->{job}{progress}, "job_0069");
}
}
else
{
# No match
$anvil->data->{job}{progress} += 3;
update_progress($anvil, $anvil->data->{job}{progress}, "job_0069");
}
}
return(0);

Loading…
Cancel
Save