This adds the new 'striker-collect-debug' tool that collects all potentially useful debug info into a single tarball.
* Fixed a bug in Get->anvil_from_switch() to work when the Anvil! name is passed. Signed-off-by: digimer <mkelly@alteeve.ca>main
digimer
1 year ago
parent
bf1ccc8bee
commit
a7ebe45f76
9 changed files with 1037 additions and 121 deletions
@ -0,0 +1,41 @@ |
||||
.\" Manpage for the Anvil! machine power and access reporting tool. |
||||
.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions. |
||||
.TH striker-collect-data "8" "July 04 2023" "Anvil! Intelligent Availability™ Platform" |
||||
.SH NAME |
||||
striker-collect-data \- This program collects data needed to help diagnose problems with an Anvil! system. |
||||
.SH SYNOPSIS |
||||
.B striker-collect-data |
||||
\fI\,<command> \/\fR[\fI\,options\/\fR] |
||||
.SH DESCRIPTION |
||||
This program collects database data, logs, config files and other information needed to help diagnose problems with the Anvil! platform. By default, this collects all data from all accessible machines. |
||||
.TP |
||||
.B Note: |
||||
.TP |
||||
This program collects potentially secure information, like passwords. Be careful who you share the collected data with! |
||||
.TP |
||||
The data from Striker dashboards are always collected. |
||||
.TP |
||||
.SH OPTIONS |
||||
.TP |
||||
\-?, \-h, \fB\-\-help\fR |
||||
Show this man page. |
||||
.TP |
||||
\fB\-\-log-secure\fR |
||||
When logging, record sensitive data, like passwords. |
||||
.TP |
||||
\-v, \-vv, \-vvv |
||||
Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data. |
||||
.SS "Commands:" |
||||
.TP |
||||
\fB\-\-anvil\fR <name or UUID> |
||||
.TP |
||||
This restricts the data to be collected to the Striker dashboards and the specific Anvil! node pair. |
||||
.TP |
||||
\fB\-\-hosts\fR <comma-separated list of host names or UUIDs> |
||||
.TP |
||||
This can be used to specify which specific hosts data is collected from. Note that this can be used in conjuction with \fB\-\-anvil\fR to add additional hosts to collect data from, like DR hosts. |
||||
.IP |
||||
.SH AUTHOR |
||||
Written by Madison Kelly, Alteeve staff and the Anvil! project contributors. |
||||
.SH "REPORTING BUGS" |
||||
Report bugs to users@clusterlabs.org |
||||
@ -0,0 +1,737 @@ |
||||
#!/usr/bin/perl |
||||
# |
||||
# This program will collect data from all accessible machines and compile it into a common tarball. This is |
||||
# designed to make it easier to diagnose faults. |
||||
# |
||||
# Exit codes; |
||||
# 0 = Normal exit. |
||||
# 1 = No database connection. |
||||
# |
||||
# TODO: |
||||
# |
||||
# USAGE: |
||||
# |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use Anvil::Tools; |
||||
require POSIX; |
||||
use Term::Cap; |
||||
use Text::Diff; |
||||
use Data::Dumper; |
||||
|
||||
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; |
||||
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; |
||||
if (($running_directory =~ /^\./) && ($ENV{PWD})) |
||||
{ |
||||
$running_directory =~ s/^\./$ENV{PWD}/; |
||||
} |
||||
|
||||
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. |
||||
$| = 1; |
||||
|
||||
my $anvil = Anvil::Tools->new(); |
||||
|
||||
### TODO: Remove this before final release |
||||
$anvil->Log->level({set => 2}); |
||||
$anvil->Log->secure({set => 1}); |
||||
########################################## |
||||
|
||||
# Read switches (target ([user@]host[:port]) and the file with the target's password. |
||||
$anvil->Get->switches({list => ["anvil", "hosts"], man => $THIS_FILE}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); |
||||
|
||||
# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks |
||||
# is to setup the database server. |
||||
$anvil->Database->connect(); |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); |
||||
if (not $anvil->data->{sys}{database}{connections}) |
||||
{ |
||||
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try |
||||
# again after we exit. |
||||
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0305"}); |
||||
sleep 10; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
# Make sure we're running as 'root' |
||||
# $< == real UID, $> == effective UID |
||||
if (($< != 0) && ($> != 0)) |
||||
{ |
||||
# Not root |
||||
print $anvil->Words->string({key => "error_0005"})."\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
# Make sure we're a striker. |
||||
if ($anvil->Get->host_type ne "striker") |
||||
{ |
||||
print "This has to be run on a Striker dashboard.\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
print "Data collection has begun.\n"; |
||||
print "Depending on how many systems we're collecting from, this could take a while.\n"; |
||||
|
||||
process_switches($anvil); |
||||
|
||||
collect_data($anvil); |
||||
|
||||
# Create the tarball now. |
||||
print "Data collection complete, creating the tarball now... "; |
||||
my $tarball = "/root/anvil-debug_".$anvil->data->{sys}{date_and_time}.".tar.bz2"; |
||||
my $shell_call = $anvil->data->{path}{exe}{tar}." -cvjf ".$tarball." ".$anvil->data->{sys}{compile_directory}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:tarball' => $tarball, |
||||
's2:shell_call' => $shell_call, |
||||
}}); |
||||
|
||||
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
print "Done!\n"; |
||||
|
||||
print "\n[ Complete ] - The debug data is here: [".$tarball."]\n"; |
||||
print "[ Warning ] - The collected logs likely include sensitive information! Share is carefully!\n"; |
||||
|
||||
|
||||
|
||||
$anvil->nice_exit({exit_code => 0}); |
||||
|
||||
|
||||
############################################################################################################# |
||||
# Functions # |
||||
############################################################################################################# |
||||
|
||||
sub collect_data |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
# Make sure the collection directory exists. |
||||
$anvil->data->{sys}{date_and_time} = $anvil->Get->date_and_time({file_name => 1}); |
||||
$anvil->data->{sys}{compile_directory} = "/tmp/anvil-debug_".$anvil->data->{sys}{date_and_time}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
"sys::date_and_time" => $anvil->data->{sys}{date_and_time}, |
||||
"sys::compile_directory" => $anvil->data->{sys}{compile_directory}, |
||||
}}); |
||||
|
||||
my $failed = $anvil->Storage->make_directory({directory => $anvil->data->{sys}{compile_directory}}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); |
||||
if ($failed) |
||||
{ |
||||
print "Failed to create the directory: [".$anvil->data->{sys}{compile_directory}."]. The error should be logged.\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
my $hosts = @{$anvil->data->{collect_from}}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { hosts => $hosts }}); |
||||
foreach my $host_type ("striker", "node", "dr") |
||||
{ |
||||
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}}) |
||||
{ |
||||
my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name}; |
||||
my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name}; |
||||
my $this_host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:host_name' => $host_name, |
||||
's2:host_uuid' => $host_uuid, |
||||
's3:short_host_name' => $short_host_name, |
||||
's4:this_host_type' => $this_host_type, |
||||
}}); |
||||
next if $host_type ne $this_host_type; |
||||
|
||||
# Are we collecting from a subset only? |
||||
if ($hosts) |
||||
{ |
||||
# Yes, is this host one of them? |
||||
my $found = 0; |
||||
foreach my $this_host_uuid (@{$anvil->data->{collect_from}}) |
||||
{ |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
host_uuid => $host_uuid, |
||||
this_host_uuid => $this_host_uuid, |
||||
}}); |
||||
if ($this_host_uuid eq $host_uuid) |
||||
{ |
||||
$found = 1; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { found => $found }}); |
||||
last; |
||||
} |
||||
} |
||||
next if not $found; |
||||
} |
||||
|
||||
# Make sure there's a directory for this host. |
||||
my $target_directory = $anvil->data->{sys}{compile_directory}."/".$short_host_name; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_directory => $target_directory }}); |
||||
if (not -d $target_directory) |
||||
{ |
||||
my $failed = $anvil->Storage->make_directory({ |
||||
directory => $target_directory, |
||||
mode => "777", |
||||
}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }}); |
||||
if ($failed) |
||||
{ |
||||
print "Failed to create the directory: [".$target_directory."]. The error should be logged.\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
} |
||||
|
||||
# Is this the local host or a remote one? |
||||
if ($host_uuid eq $anvil->Get->host_uuid) |
||||
{ |
||||
### Collecting local data. |
||||
collect_local_data($anvil, $target_directory); |
||||
} |
||||
else |
||||
{ |
||||
# Collecting data from a remote machine |
||||
my $problem = collect_remote_data($anvil, $host_uuid, $target_directory); |
||||
if ($problem) |
||||
{ |
||||
# Create a file saying we couldn't access this machine. |
||||
my $body = "No access to: [".$host_name."] found.\n"; |
||||
my $file = $target_directory."/no_access.txt"; |
||||
$anvil->Storage->write_file({ |
||||
file => $file, |
||||
body => $body, |
||||
overwrite => 1, |
||||
backup => 0, |
||||
}); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
sub collect_remote_data |
||||
{ |
||||
my ($anvil, $host_uuid, $target_directory) = @_; |
||||
|
||||
my $host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_name}; |
||||
my $short_host_name = $anvil->data->{hosts}{host_uuid}{$host_uuid}{short_host_name}; |
||||
my $this_host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}; |
||||
my $failed_body = "File not copied from: [".$host_name."].\n"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:host_name' => $host_name, |
||||
's2:host_uuid' => $host_uuid, |
||||
's3:short_host_name' => $short_host_name, |
||||
's4:this_host_type' => $this_host_type, |
||||
's5:target_directory' => $target_directory, |
||||
}}); |
||||
|
||||
# Dump the previous boot logs to a file. |
||||
print "\nGrabbing logs and data from the remote system: [".$short_host_name."].\n"; |
||||
print "- Testing access...\n"; |
||||
my $matches = $anvil->Network->find_access({ |
||||
debug => 2, |
||||
target => $host_name, |
||||
}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { matches => $matches }}); |
||||
$anvil->data->{peer}{$short_host_name}{access}{ip} = ""; |
||||
$anvil->data->{peer}{$short_host_name}{access}{network} = ""; |
||||
foreach my $preferred_network ("bcn", "mn", "ifn", "sn") |
||||
{ |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { preferred_network => $preferred_network }}); |
||||
foreach my $network_name (sort {$a cmp $b} keys %{$anvil->data->{network_access}}) |
||||
{ |
||||
next if $network_name !~ /^$preferred_network/; |
||||
my $target_ip = $anvil->data->{network_access}{$network_name}{target_ip_address}; |
||||
my $test_access = $anvil->Remote->test_access({target => $target_ip}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:network_name' => $network_name, |
||||
's2:target_ip' => $target_ip, |
||||
's3:test_access' => $test_access, |
||||
}}); |
||||
|
||||
if ($test_access) |
||||
{ |
||||
# We're good. |
||||
print "- Found access over the network: [".$network_name."] using the target IP: [".$target_ip."]\n"; |
||||
$anvil->data->{peer}{$short_host_name}{access}{ip} = $target_ip; |
||||
$anvil->data->{peer}{$short_host_name}{access}{network} = $network_name; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
"s1:peer::${short_host_name}::access::ip" => $anvil->data->{peer}{$short_host_name}{access}{ip}, |
||||
"s2:peer::${short_host_name}::access::network" => $anvil->data->{peer}{$short_host_name}{access}{network}, |
||||
}}); |
||||
} |
||||
} |
||||
} |
||||
|
||||
if (not $anvil->data->{peer}{$short_host_name}{access}{ip}) |
||||
{ |
||||
print "No access!!\n"; |
||||
print "- Not able to collect data from this host, skipping.\n"; |
||||
return(1); |
||||
} |
||||
|
||||
print "- Writing out system logs from the previous boot... "; |
||||
my $shell_call = $anvil->data->{path}{exe}{journalctl}." -b -1 > /tmp/journalctl-previous-boot.log"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
my ($output, $error, $return_code) = $anvil->Remote->call({ |
||||
shell_call => $shell_call, |
||||
target => $anvil->data->{peer}{$short_host_name}{access}{ip}, |
||||
}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
error => $error, |
||||
return_code => $return_code, |
||||
}}); |
||||
|
||||
# Copying the file |
||||
print "Done! Copying to here... "; |
||||
$anvil->Storage->rsync({ |
||||
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/tmp/journalctl-previous-boot.log", |
||||
destination => $target_directory."/", |
||||
}); |
||||
my $test_file = $target_directory."/tmp/journalctl-previous-boot.log"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }}); |
||||
if (not -e $test_file) |
||||
{ |
||||
print "Done.\n"; |
||||
} |
||||
else |
||||
{ |
||||
print "Failed!\n"; |
||||
print "- For some reason, this file was not collected.\n"; |
||||
$anvil->Storage->write_file({ |
||||
file => $test_file, |
||||
body => $failed_body, |
||||
overwrite => 1, |
||||
backup => 0, |
||||
}); |
||||
} |
||||
|
||||
# Dump the current boot logs |
||||
print "- Grabbing system logs from this boot... "; |
||||
$shell_call = $anvil->data->{path}{exe}{journalctl}." -b 0 > /tmp/journalctl-current-boot.log"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
|
||||
# Copying the file |
||||
print "Done! Copying to here... "; |
||||
$anvil->Storage->rsync({ |
||||
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/tmp/journalctl-current-boot.log", |
||||
destination => $target_directory."/", |
||||
}); |
||||
$test_file = $target_directory."/journalctl-current-boot.log"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }}); |
||||
if (not -e $test_file) |
||||
{ |
||||
print "Done.\n"; |
||||
} |
||||
else |
||||
{ |
||||
print "Failed!\n"; |
||||
print "- For some reason, this file was not collected.\n"; |
||||
$anvil->Storage->write_file({ |
||||
file => $test_file, |
||||
body => $failed_body, |
||||
overwrite => 1, |
||||
backup => 0, |
||||
}); |
||||
} |
||||
|
||||
# If we're a striker, dump the database also. |
||||
if ($this_host_type eq "striker") |
||||
{ |
||||
# What's the password and address? |
||||
if (not exists $anvil->data->{database}{$host_uuid}) |
||||
{ |
||||
# The remote striker isn't known |
||||
print "- The host is a Striker, but we don't have database access info, skipping DB dump.\n"; |
||||
} |
||||
else |
||||
{ |
||||
print "- Dumping and compressing remote database data, PLEASE BE PATIENT!... "; |
||||
my $pg_file = "/root/.pgpass"; |
||||
my $pg_body = "*:*:*:admin:".$anvil->data->{database}{$host_uuid}{password}; |
||||
$anvil->Storage->write_file({ |
||||
file => $pg_file, |
||||
body => $pg_body, |
||||
mode => "600", |
||||
overwrite => 0, |
||||
backup => 0, |
||||
}); |
||||
my $shell_call = $anvil->data->{path}{exe}{pg_dump}." -h ".$anvil->data->{peer}{$short_host_name}{access}{ip}." -U admin anvil 2>/dev/null | ".$anvil->data->{path}{exe}{bzip2}." --stdout > ".$target_directory."/anvil.out.bz2"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
if ($return_code) |
||||
{ |
||||
# Failed |
||||
print "Failed!\n"; |
||||
print "Expected the return code '0', but got: [".$return_code."]. The error, if any, was:\n"; |
||||
print "========\n"; |
||||
print $output."\n"; |
||||
print "========\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
unlink $pg_file; |
||||
print "Done!\n"; |
||||
} |
||||
} |
||||
|
||||
print "- Grabbing hosts file... "; |
||||
$anvil->Storage->rsync({ |
||||
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/etc/hosts", |
||||
destination => $target_directory."/", |
||||
}); |
||||
$test_file = $target_directory."/hosts"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }}); |
||||
if (not -e $test_file) |
||||
{ |
||||
print "Done.\n"; |
||||
} |
||||
else |
||||
{ |
||||
print "Failed!\n"; |
||||
print "- For some reason, this file was not collected.\n"; |
||||
$anvil->Storage->write_file({ |
||||
file => $test_file, |
||||
body => $failed_body, |
||||
overwrite => 1, |
||||
backup => 0, |
||||
}); |
||||
} |
||||
|
||||
print "- Grabbing Anvil! log... "; |
||||
$anvil->Storage->rsync({ |
||||
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/var/log/anvil.log", |
||||
destination => $target_directory."/", |
||||
}); |
||||
$test_file = $target_directory."/anvil.log"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }}); |
||||
if (not -e $test_file) |
||||
{ |
||||
print "Done.\n"; |
||||
} |
||||
else |
||||
{ |
||||
print "Failed!\n"; |
||||
print "- For some reason, this file was not collected.\n"; |
||||
$anvil->Storage->write_file({ |
||||
file => $test_file, |
||||
body => $failed_body, |
||||
overwrite => 1, |
||||
backup => 0, |
||||
}); |
||||
} |
||||
|
||||
# If this is a node, grab the shared files. |
||||
if ($this_host_type eq "node") |
||||
{ |
||||
print "- Collecting the cluster information base (CIB)... "; |
||||
$shell_call = $anvil->data->{path}{exe}{pcs}." cluster cib > /tmp/cib.xml"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
|
||||
# Copying the file |
||||
print "Done! Copying to here... "; |
||||
$anvil->Storage->rsync({ |
||||
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/tmp/cib.xml", |
||||
destination => $target_directory."/", |
||||
}); |
||||
my $test_file = $target_directory."/cib.xml"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }}); |
||||
if (not -e $test_file) |
||||
{ |
||||
print "Done.\n"; |
||||
} |
||||
else |
||||
{ |
||||
print "Failed!\n"; |
||||
print "- For some reason, this file was not collected.\n"; |
||||
$anvil->Storage->write_file({ |
||||
file => $test_file, |
||||
body => $failed_body, |
||||
overwrite => 1, |
||||
backup => 0, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
# If this is not a striker, collect definition files. |
||||
if ($this_host_type ne "striker") |
||||
{ |
||||
print "- Collecting server definitions... "; |
||||
$anvil->Storage->rsync({ |
||||
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/mnt/shared/definitions", |
||||
destination => $target_directory."/", |
||||
}); |
||||
print "Done!\n"; |
||||
|
||||
print "- Collecting replicated storage config... "; |
||||
$anvil->Storage->rsync({ |
||||
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/etc/drbd.d", |
||||
destination => $target_directory."/", |
||||
}); |
||||
print "Done!\n"; |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
sub collect_local_data |
||||
{ |
||||
my ($anvil, $target_directory) = @_; |
||||
|
||||
my $host_uuid = $anvil->Get->host_uuid(); |
||||
my $this_host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:target_directory' => $target_directory, |
||||
's2:host_uuid' => $host_uuid, |
||||
's3:this_host_type' => $this_host_type, |
||||
}}); |
||||
|
||||
# Dump the previous boot logs to a file. |
||||
print "\nGrabbing logs and data from the local system.\n"; |
||||
print "- Grabbing system logs from the previous boot... "; |
||||
my $shell_call = $anvil->data->{path}{exe}{journalctl}." -b -1 > ".$target_directory."/journalctl-previous-boot.log"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
print "Done!\n"; |
||||
|
||||
# Dump the current boot logs |
||||
print "- Grabbing system logs from this boot... "; |
||||
$shell_call = $anvil->data->{path}{exe}{journalctl}." -b 0 > ".$target_directory."/journalctl-current-boot.log"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
print "Done!\n"; |
||||
|
||||
# If we're a striker, dump the database also. |
||||
if ($this_host_type eq "striker") |
||||
{ |
||||
print "- Dumping and compressing database data, PLEASE BE PATIENT!... "; |
||||
my $shell_call = $anvil->data->{path}{exe}{su}." postgres -c \"".$anvil->data->{path}{exe}{pg_dump}." anvil\" 2>/dev/null | ".$anvil->data->{path}{exe}{bzip2}." --stdout > ".$target_directory."/anvil.out.bz2"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
if ($return_code) |
||||
{ |
||||
# Failed |
||||
print "Failed!\n"; |
||||
print "Expected the return code '0', but got: [".$return_code."]. The error, if any, was:\n"; |
||||
print "========\n"; |
||||
print $output."\n"; |
||||
print "========\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
print "Done!\n"; |
||||
} |
||||
|
||||
print "- Grabbing hosts file... "; |
||||
$shell_call = $anvil->data->{path}{exe}{cp}." /etc/hosts ".$target_directory."/"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
print "Done!\n"; |
||||
|
||||
print "- Grabbing Anvil! log... "; |
||||
$shell_call = $anvil->data->{path}{exe}{cp}." /var/log/anvil.log ".$target_directory."/"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
print "Done!\n"; |
||||
|
||||
# If this is a node, grab the shared files. |
||||
if ($this_host_type eq "node") |
||||
{ |
||||
print "- Collecting the cluster information base (CIB)... "; |
||||
$shell_call = $anvil->data->{path}{exe}{pcs}." cluster cib > ".$target_directory."/cib.xml"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
print "Done!\n"; |
||||
} |
||||
|
||||
# If this is not a striker, collect definition files. |
||||
if ($this_host_type ne "striker") |
||||
{ |
||||
print "- Collecting server definitions... "; |
||||
$shell_call = $anvil->data->{path}{exe}{rsync}." -av /mnt/shared/definitions ".$target_directory."/"; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); |
||||
|
||||
($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
output => $output, |
||||
return_code => $return_code, |
||||
}}); |
||||
print "Done!\n"; |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
|
||||
sub process_switches |
||||
{ |
||||
my ($anvil) = @_; |
||||
|
||||
$anvil->data->{collect_from} = []; |
||||
$anvil->Database->get_hosts(); |
||||
|
||||
if ($anvil->data->{switches}{anvil}) |
||||
{ |
||||
if ($anvil->data->{switches}{anvil} eq "#!SET!#") |
||||
{ |
||||
# Show a list of Anvil! systems. |
||||
print "Available Anvil! systems. Use '--anvil <name or UUID>' to collect data from a specific Anvil! node.\n"; |
||||
foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}}) |
||||
{ |
||||
print "- Name: [".$anvil_name."], UUID: [".$anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid}."]\n"; |
||||
} |
||||
$anvil->nice_exit({exit_code => 0}); |
||||
} |
||||
|
||||
# Make sure the anvil is valid. |
||||
my ($anvil_name, $anvil_uuid) = $anvil->Get->anvil_from_switch({ |
||||
debug => 2, |
||||
anvil => $anvil->data->{switches}{anvil}, |
||||
}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:anvil_name' => $anvil_name, |
||||
's2:anvil_uuid' => $anvil_uuid, |
||||
}}); |
||||
|
||||
if (not $anvil_name) |
||||
{ |
||||
# Bad name. |
||||
print "[ Error ] - Unable to get the Anvil! name and UUID from the string: [".$anvil->data->{switches}{anvil}."]\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
|
||||
# Add the host_uuids to the collect_from array. |
||||
push @{$anvil->data->{collect_from}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; |
||||
push @{$anvil->data->{collect_from}}, $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; |
||||
} |
||||
|
||||
if ($anvil->data->{switches}{hosts}) |
||||
{ |
||||
if ($anvil->data->{switches}{hosts} eq "#!SET!#") |
||||
{ |
||||
# Show a list of all machines. |
||||
print "Available Anvil! cluster systems. Use '--host <comma-separated list of names or UUIDs>' to collect data from specific hosts.\n"; |
||||
foreach my $host_type ("striker", "node", "dr") |
||||
{ |
||||
print "- Striker Dashboards:\n" if $host_type eq "striker"; |
||||
print "\n- Anvil! sub-nodes:\n" if $host_type eq "node"; |
||||
print "\n- Disaster recovery hosts:\n" if $host_type eq "dr"; |
||||
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}}) |
||||
{ |
||||
my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name}; |
||||
my $this_host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:host_name' => $host_name, |
||||
's2:host_uuid' => $host_uuid, |
||||
's3:this_host_type' => $this_host_type, |
||||
}}); |
||||
next if $host_type ne $this_host_type; |
||||
|
||||
print " - Host: [".$host_name."], UUID: [".$host_uuid."]\n"; |
||||
} |
||||
} |
||||
|
||||
$anvil->nice_exit({exit_code => 0}); |
||||
} |
||||
|
||||
foreach my $host (split/,/, $anvil->data->{switches}{hosts}) |
||||
{ |
||||
# Make sure this host is valid. |
||||
my ($host_uuid) = $anvil->Database->get_host_uuid_from_string({string => $host}); |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:host' => $host, |
||||
's2:host_uuid' => $host_uuid, |
||||
}}); |
||||
if (not $host_uuid) |
||||
{ |
||||
print "[ Error ] - Unable to get the host UUID from the host string: [".$host."]\n"; |
||||
$anvil->nice_exit({exit_code => 1}); |
||||
} |
||||
push @{$anvil->data->{collect_from}}, $host_uuid; |
||||
} |
||||
} |
||||
|
||||
# If we were restricted to an anvil or host, make sure we've added the Strikers. |
||||
if (($anvil->data->{switches}{anvil}) or ($anvil->data->{switches}{hosts})) |
||||
{ |
||||
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{sys}{hosts}{by_name}}) |
||||
{ |
||||
my $host_uuid = $anvil->data->{sys}{hosts}{by_name}{$host_name}; |
||||
my $this_host_type = $anvil->data->{hosts}{host_uuid}{$host_uuid}{host_type}; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:host_name' => $host_name, |
||||
's2:host_uuid' => $host_uuid, |
||||
's3:this_host_type' => $this_host_type, |
||||
}}); |
||||
next if $this_host_type ne "striker"; |
||||
|
||||
my $seen = 0; |
||||
foreach my $this_host_uuid (@{$anvil->data->{collect_from}}) |
||||
{ |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { |
||||
's1:this_host_uuid' => $this_host_uuid, |
||||
's2:host_uuid' => $host_uuid, |
||||
}}); |
||||
if ($this_host_uuid eq $host_uuid) |
||||
{ |
||||
$seen = 1; |
||||
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { seen => $seen }}); |
||||
last; |
||||
} |
||||
} |
||||
|
||||
if (not $seen) |
||||
{ |
||||
push @{$anvil->data->{collect_from}}, $host_uuid; |
||||
} |
||||
} |
||||
} |
||||
|
||||
return(0); |
||||
} |
||||
Loading…
Reference in new issue