seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Made more progress on the resync code.

Browse Source 
* Added notes on SQL table requirements.
* Added the ability to sort columns in Log->variables.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 7 years ago
parent 21694716e8
commit 1aefa7fb8a
3 changed files with 159 additions and 22 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 19
      AN/Tools.sql
  2. 123
      AN/Tools/Database.pm
  3. 37
      AN/Tools/Log.pm

19
AN/Tools.sql
Unescape View File

@ -1,6 +1,24 @@
-- This is the core database schema for AN::Tools. -- This is the core database schema for AN::Tools.
-- It expects PostgreSQL v. 9.1+ -- It expects PostgreSQL v. 9.1+
-- Table construction rules;
--
-- All tables need to have a column called '<table>_uuid uuid not null primary key' that will have a
-- unique UUID. This is used to keep track of the same entry in the history schema. If the table ends in a
-- plural, the '<table>_uuid' and can use the singular form of the table. For example, the table 'hosts' can
-- use 'host_uuid'.
--
-- All tables must hast a 'modified_date timestamp with time zone not null' column. This is used to track
-- changes through time in the history schema and used to groups changes when resync'ing.
--
-- Tables can optionally have a '*_host_uuid uuid not null' colum. If this is found, when resync'ing the
-- table, the resync will be restricted to the host's 'sys::host_uuid'.
--
-- Most tables will want to have a matching table in the history schema with an additional
-- 'history_id bigserial' column. Match the function and trigger seen elsewhere to copy your data from the
-- public schema to the history schema on UPDATE or INSERT.
SET client_encoding = 'UTF8'; SET client_encoding = 'UTF8';
-- This doesn't work before 9.3 - CREATE SCHEMA IF NOT EXISTS history; -- This doesn't work before 9.3 - CREATE SCHEMA IF NOT EXISTS history;
-- So we'll use the query below until (if) we upgrade. -- So we'll use the query below until (if) we upgrade.
@ -271,6 +289,7 @@ ALTER TABLE updated OWNER TO #!variable!user!#;
-- previously pass the "alert" threshold, we DON'T want to send an "all clear" message. So do solve that, -- previously pass the "alert" threshold, we DON'T want to send an "all clear" message. So do solve that,
-- this table is used by agents to record when a warning message was sent. -- this table is used by agents to record when a warning message was sent.
CREATE TABLE alert_sent ( CREATE TABLE alert_sent (
alert_sent_uuid uuid primary key,
alert_sent_host_uuid uuid not null, -- The node associated with this alert alert_sent_host_uuid uuid not null, -- The node associated with this alert
alert_set_by text not null, -- name of the program that set this alert alert_set_by text not null, -- name of the program that set this alert
alert_record_locator text not null, -- String used by the agent to identify the source of the alert (ie: UPS serial number) alert_record_locator text not null, -- String used by the agent to identify the source of the alert (ie: UPS serial number)

123
AN/Tools/Database.pm
Unescape View File

@ -2508,7 +2508,8 @@ sub resync_databases
### NOTE: Don't sort this array, we need to resync in the order that the user passed the tables to us ### NOTE: Don't sort this array, we need to resync in the order that the user passed the tables to us
### to avoid trouble with primary/foreign keys. ### to avoid trouble with primary/foreign keys.
# We're going to use the array of tables assembles by _find_behind_databases() stored in 'sys::database::check_tables' # We're going to use the array of tables assembles by _find_behind_databases() stored in
# 'sys::database::check_tables'
foreach my $table (@{$an->data->{sys}{database}{check_tables}}) foreach my $table (@{$an->data->{sys}{database}{check_tables}})
{ {
# If the 'schema' is 'public', there is no table in the history schema. If there is a host # If the 'schema' is 'public', there is no table in the history schema. If there is a host
@ -2521,9 +2522,29 @@ sub resync_databases
host_column => $host_column, host_column => $host_column,
}}); }});
# If there is a column name that is '<table>_uuid', or the same with the table's name minus
# the last 's', this will be the UUID column to keep records linked in history. We'll need to
# know this off the bat. Tables where we don't find a UUID column won't be sync'ed.
my $column1 = $table."_uuid";
my $column2 = "";
if ($table =~ /^(.*)s$/)
{
$column2 = $1."_uuid";
}
my $query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'scancore' AND table_schema = 'public' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table)." AND data_type = 'uuid' AND is_nullable = 'NO' AND column_name = ".$an->data->{sys}{use_db_fh}->quote($column1).";";
if ($column2)
{
$query = "SELECT column_name FROM information_schema.columns WHERE table_catalog = 'scancore' AND table_schema = 'public' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table)." AND data_type = 'uuid' AND is_nullable = 'NO' AND (column_name = ".$an->data->{sys}{use_db_fh}->quote($column1)." OR column_name = ".$an->data->{sys}{use_db_fh}->quote($column2).");";
}
$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $uuid_column = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$uuid_column = "" if not defined $uuid_column;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uuid_column => $uuid_column }});
next if not $uuid_column;
# Get all the columns in this table. # Get all the columns in this table.
my $query = "SELECT column_name, is_nullable, data_type FROM information_schema.columns WHERE table_schema = ".$an->data->{sys}{use_db_fh}->quote($schema)." AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table)." AND column_name != 'history_id';"; $query = "SELECT column_name, is_nullable, data_type FROM information_schema.columns WHERE table_schema = ".$an->data->{sys}{use_db_fh}->quote($schema)." AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table)." AND column_name != 'history_id';";
$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }}); $an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0124", variables => { query => $query }});
my $results = $an->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $an->Database->query({query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
@ -2558,33 +2579,33 @@ sub resync_databases
$an->data->{db_resync}{$id}{public}{sql} = []; $an->data->{db_resync}{$id}{public}{sql} = [];
$an->data->{db_resync}{$id}{history}{sql} = []; $an->data->{db_resync}{$id}{history}{sql} = [];
# Read in the history schema # Read in the data, modified_date first as we'll need that for all entries we record.
my $query = "SELECT "; my $query = "SELECT modified_date, $uuid_column, ";
my $read_columns = []; my $read_columns = [];
push @{$read_columns}, "modified_date";
foreach my $column_name (sort {$a cmp $b} keys %{$an->data->{sys}{database}{table}{$table}{column}}) foreach my $column_name (sort {$a cmp $b} keys %{$an->data->{sys}{database}{table}{$table}{column}})
{ {
# We'll skip the host column as we'll use it in the conditional. # We'll skip the host column as we'll use it in the conditional.
next if $column_name eq $host_column;
next if $column_name eq "modified_date"; next if $column_name eq "modified_date";
next if $column_name eq $host_column;
next if $column_name eq $uuid_column;
$query .= $column_name.", "; $query .= $column_name.", ";
push @{$read_columns}, $column_name; push @{$read_columns}, $column_name;
} }
# Manually add modified_date. # Strip the last comma and the add the schema.table name.
push @{$read_columns}, "modified_date"; $query =~ s/, $/ /;
$query .= "modified_date FROM ".$schema.".".$table." "; $query .= "FROM ".$schema.".".$table;
# Restrict to this host if a host column was found. # Restrict to this host if a host column was found.
if ($host_column) if ($host_column)
{ {
$query .= "WHERE ".$host_column." = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{host_uuid}); $query .= " WHERE ".$host_column." = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{host_uuid});
} }
$query .= ";"; $query .= ";";
$an->Log->entry({level => 2, key => "log_0074", variables => { id => $id, query => $query }}); $an->Log->entry({level => 2, key => "log_0074", variables => { id => $id, query => $query }});
$an->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, key => "log_0124", variables => { query => $query }});
my $results = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__}); my $results = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__});
my $count = @{$results}; my $count = @{$results};
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
@ -2595,6 +2616,8 @@ sub resync_databases
foreach my $row (@{$results}) foreach my $row (@{$results})
{ {
my $modified_date = "";
my $row_uuid = "";
for (my $i = 0; $i < @{$read_columns}; $i++) for (my $i = 0; $i < @{$read_columns}; $i++)
{ {
my $column_name = $read_columns->[$i]; my $column_name = $read_columns->[$i];
@ -2602,16 +2625,80 @@ sub resync_databases
my $not_null = $an->data->{sys}{database}{table}{$table}{column}{$column_name}{not_null}; my $not_null = $an->data->{sys}{database}{table}{$table}{column}{$column_name}{not_null};
my $data_type = $an->data->{sys}{database}{table}{$table}{column}{$column_name}{data_type}; my $data_type = $an->data->{sys}{database}{table}{$table}{column}{$column_name}{data_type};
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
column_name => $column_name, "s1:i" => $i,
column_value => $column_value, "s2:column_name" => $column_name,
not_null => $not_null, "s3:column_value" => $column_value,
data_type => $data_type, "s4:not_null" => $not_null,
"s5:data_type" => $data_type,
}}); }});
if ((not $not_null) && ($column_value eq "NULL")) if ((not $not_null) && ($column_value eq "NULL"))
{ {
$column_value = ""; $column_value = "";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column_value => $column_value }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { column_value => $column_value }});
} }
# The modified_date should be the first row.
if ($column_name eq "modified_date")
{
$modified_date = $column_value;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { modified_date => $modified_date }});
next;
}
# The row's UUID should be the second row.
if ($column_name eq $uuid_column)
{
$row_uuid = $column_value;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { row_uuid => $row_uuid }});
# This is used to determine if a given entry needs to be
# updated or inserted into the public schema
$an->data->{db_data}{$id}{$table}{$uuid_column}{$row_uuid}{'exists'} = 1;
$an->data->{db_data}{$id}{$table}{$uuid_column}{$row_uuid}{seen} = 0;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"db_data::${id}::${table}::${uuid_column}::${row_uuid}::exists" => $an->data->{db_data}{$id}{$table}{$uuid_column}{$row_uuid}{'exists'},
"db_data::${id}::${table}::${uuid_column}::${row_uuid}::seen" => $an->data->{db_data}{$id}{$table}{$uuid_column}{$row_uuid}{seen},
}});
next;
}
die $THIS_FILE." ".__LINE__."; This row's modified_date wasn't the first column returned in query: [$query]\n" if not $modified_date;
die $THIS_FILE." ".__LINE__."; This row's UUID column: [$uuid_column] wasn't the second column returned in query: [$query]\n" if not $modified_date;
# Record this in the unified and local hashes. Note that we'll handle
# the 'hosts' table in a special way, then the rest depending on
# whether we have a host column or not.
if ($host_column)
{
# We habe a host column
}
else
{
# This table isn't restricted to given hosts.
$an->data->{db_data}{unified}{hosts}{modified_date}{$modified_date}{$uuid_column}{$row_uuid}{$column_name} = $column_value;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"db_data::unified::hosts::modified_date::${modified_date}::${uuid_column}::${row_uuid}::${column_name}" => $an->data->{db_data}{unified}{hosts}{modified_date}{$modified_date}{$uuid_column}{$row_uuid}{$column_name},
}});
}
# $an->data->{db_data}{unified}{hosts}{modified_date}{$modified_date}{host_uuid}{$host_uuid} = {
# host_location_uuid => $host_location_uuid,
# host_name => $host_name,
# host_type => $host_type,
# host_emergency_stop => $host_emergency_stop,
# host_stop_reason => $host_stop_reason,
# host_health => $host_health,
# };
#
# $an->data->{db_data}{$id}{hosts}{host_uuid}{$host_uuid}{'exists'} = 1;
# $an->data->{db_data}{$id}{hosts}{host_uuid}{$host_uuid}{seen} = 0;
# $an->data->{db_data}{$id}{hosts}{modified_date}{$modified_date}{host_uuid}{$host_uuid} = {
# host_location_uuid => $host_location_uuid,
# host_name => $host_name,
# host_type => $host_type,
# host_emergency_stop => $host_emergency_stop,
# host_stop_reason => $host_stop_reason,
# host_health => $host_health,
# };
} }
} }
die; die;
@ -2990,6 +3077,9 @@ sub _find_behind_databases
my $query = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND column_name LIKE '\%_host_uuid' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";"; my $query = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND column_name LIKE '\%_host_uuid' AND table_name = ".$an->data->{sys}{use_db_fh}->quote($table).";";
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }});
# See if there is a column that ends in '_host_uuid'. If there is, we'll use
# it later to restrict resync activity to these columns with the local
# 'sys::host_uuid'.
my $host_column = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; my $host_column = $an->Database->query({id => $id, query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$host_column = "" if not defined $host_column; $host_column = "" if not defined $host_column;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_column => $host_column }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_column => $host_column }});
@ -3002,7 +3092,6 @@ sub _find_behind_databases
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { count => $count }}); $an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { count => $count }});
my $schema = $count ? "history" : "public"; my $schema = $count ? "history" : "public";
$query = " $query = "
SELECT SELECT
round(extract(epoch from modified_date)) round(extract(epoch from modified_date))

37
AN/Tools/Log.pm
Unescape View File

@ -494,6 +494,29 @@ head3 list (required)
This is a hash reference containing the variables to record. This is a hash reference containing the variables to record.
If the passed in number of entries is 5 or less, the output will all be on one line. If more entries are passed, the variable/value pairs will be presented as one entry per line.
To allow for sorting, if the key starts with 's#:', that part of the key will be removed in the log. For example;
$an->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:i" => $i,
"s2:column_name" => $column_name,
"s3:column_value" => $column_value,
"s4:not_null" => $not_null,
"s5:data_type" => $data_type,
}});
Would generate a sorted log entry that looks like:
Aug 20 13:10:28 m3-striker01.alteeve.com an-tools[9445]: Database.pm:2604; Variables:
|- i: [0]
|- column_name: [host_name]
|- column_value: [m3-striker01.alteeve.com]
|- not_null: [1]
\- data_type: [text]
All other key names are left alone and output is sorted alphabetically.
=cut =cut
sub variables sub variables
{ {
@ -545,12 +568,15 @@ sub variables
} }
} }
my $raw = ""; my $raw = "";
if ($entries < 5) if ($entries <= 5)
{ {
# Put all the entries on one line. # Put all the entries on one line.
foreach my $key (sort {$a cmp $b} keys %{$list}) foreach my $key (sort {$a cmp $b} keys %{$list})
{ {
$raw .= "$key: [".$list->{$key}."], "; # Strip a leading 'sX:' in case the user is sorting the output.
my $say_key = $key;
$say_key =~ s/^s(\d+)://;
$raw .= "$say_key: [".$list->{$key}."], ";
} }
$raw =~ s/, $//; $raw =~ s/, $//;
} }
@ -560,13 +586,16 @@ sub variables
$raw .= $an->Words->string({key => "log_0019"})."\n"; $raw .= $an->Words->string({key => "log_0019"})."\n";
foreach my $key (sort {$a cmp $b} keys %{$list}) foreach my $key (sort {$a cmp $b} keys %{$list})
{ {
# Strip a leading 'sX:' in case the user is sorting the output.
my $say_key = $key;
$say_key =~ s/^s(\d+)://;
if ($entry ne $entries) if ($entry ne $entries)
{ {
$raw .= "|- $key: [".$list->{$key}."]\n"; $raw .= "|- $say_key: [".$list->{$key}."]\n";
} }
else else
{ {
$raw .= "\\- $key: [".$list->{$key}."]\n"; $raw .= "\\- $say_key: [".$list->{$key}."]\n";
} }
$entry++; $entry++;
} }

Write Preview
Loading…
Cancel
Save