Merge pull request #582 from ClusterLabs/anvil-tools-dev

Anvil tools dev
main
Digimer 10 months ago committed by GitHub
commit d080fbacce
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 7
      Anvil/Tools.pm
  2. 2
      Anvil/Tools/Database.pm
  3. 319
      Anvil/Tools/Get.pm
  4. 275
      Anvil/Tools/Network.pm
  5. 21
      Anvil/Tools/Remote.pm
  6. 26
      Anvil/Tools/ScanCore.pm
  7. 43
      Anvil/Tools/Storage.pm
  8. 61
      Anvil/Tools/System.pm
  9. 6
      anvil.conf
  10. 4
      anvil.spec.in
  11. 3
      man/Makefile.am
  12. 2
      man/anvil-boot-server.8
  13. 51
      man/anvil-manage-daemons.8
  14. 0
      man/anvil-monitor-network.8
  15. 41
      man/anvil-monitor-performance.8
  16. 30
      man/striker-get-screenshots.8
  17. 12
      ocf/alteeve/server
  18. 21
      share/words.xml
  19. 2
      tools/Makefile.am
  20. 2
      tools/anvil-check-memory
  21. 214
      tools/anvil-configure-host
  22. 15
      tools/anvil-daemon
  23. 4
      tools/anvil-delete-server
  24. 152
      tools/anvil-join-anvil
  25. 390
      tools/anvil-manage-daemons
  26. 159
      tools/anvil-monitor-performance
  27. 48
      tools/anvil-provision-server
  28. 5
      tools/anvil-safe-start
  29. 4
      tools/anvil-sync-shared
  30. 7
      tools/scancore
  31. 35
      tools/striker-auto-initialize-all
  32. 10
      tools/striker-collect-debug
  33. 2
      units/Makefile.am
  34. 2
      units/anvil-daemon.service
  35. 13
      units/anvil-monitor-daemons.service
  36. 12
      units/anvil-monitor-performance.service
  37. 2
      units/scancore.service

@ -908,6 +908,9 @@ sub _set_defaults
manage => { manage => {
firewall => 1, firewall => 1,
}, },
net => {
always_reconnect => 0,
},
password => { password => {
algorithm => "sha512", algorithm => "sha512",
hash_count => 500000, hash_count => 500000,
@ -1053,6 +1056,7 @@ sub _set_paths
configs => { configs => {
'alteeve-release.repo' => "/etc/yum.repos.d/alteeve-release.repo", 'alteeve-release.repo' => "/etc/yum.repos.d/alteeve-release.repo",
'anvil.conf' => "/etc/anvil/anvil.conf", 'anvil.conf' => "/etc/anvil/anvil.conf",
'anvil.debug' => "/etc/anvil/anvil.debug",
'anvil.version' => "/etc/anvil/anvil.version", 'anvil.version' => "/etc/anvil/anvil.version",
'autoindex.conf' => "/etc/httpd/conf.d/autoindex.conf", 'autoindex.conf' => "/etc/httpd/conf.d/autoindex.conf",
'cib.xml' => "/var/lib/pacemaker/cib/cib.xml", 'cib.xml' => "/var/lib/pacemaker/cib/cib.xml",
@ -1117,6 +1121,7 @@ sub _set_paths
ifcfg => "/etc/sysconfig/network-scripts", ifcfg => "/etc/sysconfig/network-scripts",
journald => "/var/log/journal", journald => "/var/log/journal",
libvirtd_definitions => "/etc/libvirt/qemu/", libvirtd_definitions => "/etc/libvirt/qemu/",
NetworkManager => "/etc/NetworkManager/",
opt_alteeve => "/opt/alteeve", opt_alteeve => "/opt/alteeve",
pgsql => "/var/lib/pgsql/", pgsql => "/var/lib/pgsql/",
resource_status => "/sys/kernel/debug/drbd/resources", resource_status => "/sys/kernel/debug/drbd/resources",
@ -1220,6 +1225,7 @@ sub _set_paths
httpd => "/usr/sbin/httpd", httpd => "/usr/sbin/httpd",
ifdown => "/sbin/ifdown", ifdown => "/sbin/ifdown",
ifup => "/sbin/ifup", ifup => "/sbin/ifup",
iostat => "/usr/bin/iostat",
ip => "/usr/sbin/ip", ip => "/usr/sbin/ip",
iperf3 => "/usr/bin/iperf3", iperf3 => "/usr/bin/iperf3",
'ipmi-oem' => "/usr/sbin/ipmi-oem", 'ipmi-oem' => "/usr/sbin/ipmi-oem",
@ -1252,6 +1258,7 @@ sub _set_paths
modprobe => "/usr/sbin/modprobe", modprobe => "/usr/sbin/modprobe",
mv => "/usr/bin/mv", mv => "/usr/bin/mv",
nc => "/usr/bin/nc", nc => "/usr/bin/nc",
'nm-online' => "/usr/bin/nm-online",
nmap => "/usr/bin/nmap", nmap => "/usr/bin/nmap",
nmcli => "/bin/nmcli", nmcli => "/bin/nmcli",
nohup => "/usr/bin/nohup", nohup => "/usr/bin/nohup",

@ -20394,8 +20394,10 @@ sub _test_access
} }
# Check using ping. Returns '1' on success, '0' on fail. # Check using ping. Returns '1' on success, '0' on fail.
alarm(120);
my $connected = $anvil->data->{cache}{database_handle}{$uuid}->ping(); my $connected = $anvil->data->{cache}{database_handle}{$uuid}->ping();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { connected => $connected }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { connected => $connected }});
alarm(0);
if (not $connected) if (not $connected)
{ {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0192", variables => { server => $say_server }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0192", variables => { server => $say_server }});

@ -33,6 +33,7 @@ my $THIS_FILE = "Get.pm";
# host_type # host_type
# host_uuid # host_uuid
# kernel_release # kernel_release
# load_average
# md5sum # md5sum
# os_type # os_type
# server_from_switch # server_from_switch
@ -2032,6 +2033,300 @@ sub kernel_release
} }
=head2 load_average
This reads in the current load average and stores the data in the following hashes;
* loads::load_average::one_minute
* loads::load_average::five_minute
* loads::load_average::ten_minute
* loads::load_average::running_processes
* loads::load_average::total_processes
This tracks the total number of interrupts since boot.
* loads::interrupts::total
This tracks the total PIDs, running PIDs and most importantly, blocked processes.
* loads::processes::total
* loads::processes::running
* loads::processes::blocked
This tracks the percentage time processes spent in certain states (see iostat);
* loads::load_percent::user
* loads::load_percent::steal
* loads::load_percent::idle
* loads::load_percent::nice
* loads::load_percent::system
* loads::load_percent::iowait
This tracks the total time spent on CPU loads, IO wait and IRQ data. Per-CPU core is tracked in matching hashes, with C<< average >> being replaced by C<< loads::cpu::core::<core_number>::X >>.
* loads::cpu::average::user_mode
* loads::cpu::average::user_mode_nice
* loads::cpu::average::system_mode
* loads::cpu::average::idle_tasks
* loads::cpu::average::io_wait
* loads::cpu::average::hard_irq
* loads::cpu::average::soft_irq
This is the number of IO operations in progress. When IOs in progress is non-zero, the weighted time (in 1/100ths of a second), doing those IOs.
* loads::storage::<device_name>::ios_currently_in_progress
* loads::storage::<device_name>::weighted_time_spent_doing_ios
This data comes from C<< /proc/loadavg >>, C<< /proc/stat >>, and C<< /proc/diskstats >>. For more information on these values, please see the relevant kernel documentation.
If there is a problem, C<< 1 >> is returned and the data in the hash will be blank, or stale. If the load averge data is collected successfully, C<< 0 >> is returned.
This method takes no parameters.
=cut
sub load_average
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Get->load_average()" }});
my $load_average = $anvil->Storage->read_file({debug => $debug, file => '/proc/loadavg' });
$load_average =~ s/\n//;
$load_average =~ s/\r//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { load_average => $load_average }});
my $stat = $anvil->Storage->read_file({debug => $debug, file => '/proc/stat' });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 'stat' => $stat }});
my $diskstats = $anvil->Storage->read_file({debug => $debug, file => '/proc/diskstats' });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { diskstats => $diskstats }});
my $shell_call = $anvil->data->{path}{exe}{iostat}." -c -o JSON ";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
my ($iostat_json, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { iostat_json => $iostat_json, return_code => $return_code }});
local $@;
my $iostat_data = "";
my $json = JSON->new->allow_nonref;
my $test = eval { $iostat_data = $json->decode($iostat_json); };
if (not $test)
{
# JSON parse failed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "error_0140", variables => {
json => $iostat_json,
error => $@,
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0519"});
return(1);
}
if (exists $anvil->data->{load_average})
{
delete $anvil->data->{load_average};
}
if (($load_average eq "!!error!!") or ($stat eq "!!error!!"))
{
return(0);
}
# Process Load Average
my ($one_minute, $five_minute, $ten_minute, $processes, $last_pid) = (split/\s+/, $load_average);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:one_minute' => $one_minute,
's2:five_minute' => $five_minute,
's3:ten_minute' => $ten_minute,
's4:processes' => $processes,
's5:last_pid' => $last_pid,
}});
my ($running_processes, $total_processes) = (split/\//, $processes);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:running_processes' => $running_processes,
's2:total_processes' => $total_processes,
}});
$anvil->data->{loads}{load_percent}{user} = $iostat_data->{sysstat}{hosts}->[0]->{statistics}->[0]->{'avg-cpu'}{user};
$anvil->data->{loads}{load_percent}{steal} = $iostat_data->{sysstat}{hosts}->[0]->{statistics}->[0]->{'avg-cpu'}{steal};
$anvil->data->{loads}{load_percent}{idle} = $iostat_data->{sysstat}{hosts}->[0]->{statistics}->[0]->{'avg-cpu'}{idle};
$anvil->data->{loads}{load_percent}{nice} = $iostat_data->{sysstat}{hosts}->[0]->{statistics}->[0]->{'avg-cpu'}{nice};
$anvil->data->{loads}{load_percent}{'system'} = $iostat_data->{sysstat}{hosts}->[0]->{statistics}->[0]->{'avg-cpu'}{'system'};
$anvil->data->{loads}{load_percent}{iowait} = $iostat_data->{sysstat}{hosts}->[0]->{statistics}->[0]->{'avg-cpu'}{iowait};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:loads::load_percent::user' => $anvil->data->{loads}{load_percent}{user}."%",
's2:loads::load_percent::steal' => $anvil->data->{loads}{load_percent}{steal}."%",
's3:loads::load_percent::idle' => $anvil->data->{loads}{load_percent}{idle}."%",
's4:loads::load_percent::nice' => $anvil->data->{loads}{load_percent}{nice}."%",
's5:loads::load_percent::system' => $anvil->data->{loads}{load_percent}{'system'}."%",
's6:loads::load_percent::iowait' => $anvil->data->{loads}{load_percent}{iowait}."%",
}});
$anvil->data->{loads}{load_average}{one_minute} = $one_minute;
$anvil->data->{loads}{load_average}{five_minute} = $five_minute;
$anvil->data->{loads}{load_average}{ten_minute} = $ten_minute;
$anvil->data->{loads}{load_average}{running_processes} = $running_processes;
$anvil->data->{loads}{load_average}{total_processes} = $total_processes;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:loads::load_average::one_minute' => $anvil->data->{loads}{load_average}{one_minute},
's2:loads::load_average::five_minute' => $anvil->data->{loads}{load_average}{five_minute},
's3:loads::load_average::ten_minute' => $anvil->data->{loads}{load_average}{ten_minute},
's4:loads::load_average::running_processes' => $anvil->data->{loads}{load_average}{running_processes},
's5:loads::load_average::total_processes' => $anvil->data->{loads}{load_average}{total_processes},
}});
### NOTE: "jiffy" = 1/100 second on x86
# Process diskstats
foreach my $line (split/\n/, $diskstats)
{
$line =~ s/^\s+//;
$line =~ s/\s+$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
### NOTE: Much of these values are not useful for us yet, as they need to be tracked over
### time to see any useful patterns. It's all parsed though, for ease of future use.
# See https://www.kernel.org/doc/Documentation/admin-guide/iostats.rst
my ($major_number,
$minor_number,
$device_name,
$reads_completed_successfully, # Since boot
$reads_merged, # How often adjacent reads are merged before being passed to the IO driver
$sectors_read, # This is the total number of sectors read successfully.
$time_spent_reading, # This is the total number of milliseconds spent by all reads
$writes_completed, # Since boot
$writes_merged, # How often adjacent writes are merged before being passed to the IO driver
$sectors_written, # This is the total number of sectors written successfully
$time_spent_writing, # This is the total number of milliseconds spent by all writes
$ios_currently_in_progress, # Number of I/Os currently in progress. - This is what we care about the most
$time_spent_doing_ios, # This field increases so long as above is non-zero. counts jiffies when at least one request was started or completed. If request runs more than 2 jiffies then some I/O time might be not accounted in case of concurrent requests
$weighted_time_spent_doing_ios, # Weighted number of milliseconds spent doing I/Os. This field is incremented at each I/O start, I/O completion, I/O merge, or read of these stats by the number of I/Os in progress times the number of milliseconds spent doing I/O since the last update of this field. This can provide an easy measure of both I/O completion time and the backlog that may be accumulating.
$discards_completed_successfully, # This is the total number of discards completed successfully.
$discards_merged, # How often adjacent discards are merged before being passed to the IO driver
$sectors_discarded, # This is the total number of sectors discarded successfully
$time_spent_discarding, # This is the total number of milliseconds spent by all discards
$flush_requests_completed_successfully, # This is the total number of flush requests completed successfully.
$time_spent_flushing # This is the total number of milliseconds spent by all flush requests.
) = (split/\s+/, $line);
# These require kernel 5.5+
$flush_requests_completed_successfully = "na" if not defined $flush_requests_completed_successfully;
$time_spent_flushing = "na" if not defined $time_spent_flushing;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's01:major_number' => $major_number,
's02:minor_number' => $minor_number,
's03:device_name' => $device_name,
's04:reads_completed_successfully' => $reads_completed_successfully,
's05:reads_merged' => $reads_merged,
's06:sectors_read' => $sectors_read,
's07:time_spent_reading' => $time_spent_reading,
's08:writes_completed' => $writes_completed,
's09:writes_merged' => $writes_merged,
's10:sectors_written' => $sectors_written,
's11:time_spent_writing' => $time_spent_writing,
's12:ios_currently_in_progress' => $ios_currently_in_progress,
's13:time_spent_doing_ios' => $time_spent_doing_ios,
's14:weighted_time_spent_doing_ios' => $weighted_time_spent_doing_ios,
's15:discards_completed_successfully' => $discards_completed_successfully,
's16:discards_merged' => $discards_merged,
's17:sectors_discarded' => $sectors_discarded,
's18:time_spent_discarding' => $time_spent_discarding,
's19:flush_requests_completed_successfully' => $flush_requests_completed_successfully,
's20:time_spent_flushing' => $time_spent_flushing,
}});
$anvil->data->{loads}{storage}{$device_name}{ios_currently_in_progress} = $ios_currently_in_progress;
$anvil->data->{loads}{storage}{$device_name}{weighted_time_spent_doing_ios} = $weighted_time_spent_doing_ios;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"loads::storage::${device_name}::ios_currently_in_progress" => $anvil->data->{loads}{storage}{$device_name}{ios_currently_in_progress},
"loads::storage::${device_name}::weighted_time_spent_doing_ios" => $anvil->data->{loads}{storage}{$device_name}{weighted_time_spent_doing_ios},
}});
}
# Process stat
foreach my $line (split/\n/, $stat)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
if ($line =~ /^cpu\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/)
{
# Time in jiffied handling tasks
$anvil->data->{loads}{cpu}{average}{user_mode} = $1;
$anvil->data->{loads}{cpu}{average}{user_mode_nice} = $2;
$anvil->data->{loads}{cpu}{average}{system_mode} = $3;
$anvil->data->{loads}{cpu}{average}{idle_tasks} = $4;
$anvil->data->{loads}{cpu}{average}{io_wait} = $5;
$anvil->data->{loads}{cpu}{average}{hard_irq} = $6;
$anvil->data->{loads}{cpu}{average}{soft_irq} = $7;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:loads::cpu::average::user_mode' => $anvil->data->{loads}{cpu}{average}{user_mode},
's2:loads::cpu::average::user_mode_nice' => $anvil->data->{loads}{cpu}{average}{user_mode_nice},
's3:loads::cpu::average::system_mode' => $anvil->data->{loads}{cpu}{average}{system_mode},
's4:loads::cpu::average::idle_tasks' => $anvil->data->{loads}{cpu}{average}{idle_tasks},
's5:loads::cpu::average::io_wait' => $anvil->data->{loads}{cpu}{average}{io_wait},
's6:loads::cpu::average::hard_irq' => $anvil->data->{loads}{cpu}{average}{hard_irq},
's7:loads::cpu::average::soft_irq' => $anvil->data->{loads}{cpu}{average}{soft_irq},
}});
}
elsif ($line =~ /^cpu(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/)
{
# Time in jiffied handling tasks
my $cpu = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { cpu => $cpu }});
$anvil->data->{loads}{cpu}{core}{$cpu}{user_mode} = $2;
$anvil->data->{loads}{cpu}{core}{$cpu}{user_mode_nice} = $3;
$anvil->data->{loads}{cpu}{core}{$cpu}{system_mode} = $4;
$anvil->data->{loads}{cpu}{core}{$cpu}{idle_tasks} = $5;
$anvil->data->{loads}{cpu}{core}{$cpu}{io_wait} = $6;
$anvil->data->{loads}{cpu}{core}{$cpu}{hard_irq} = $7;
$anvil->data->{loads}{cpu}{core}{$cpu}{soft_irq} = $8;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:loads::cpu::core::${cpu}::user_mode" => $anvil->data->{loads}{cpu}{core}{$cpu}{user_mode},
"s2:loads::cpu::core::${cpu}::user_mode_nice" => $anvil->data->{loads}{cpu}{core}{$cpu}{user_mode_nice},
"s3:loads::cpu::core::${cpu}::system_mode" => $anvil->data->{loads}{cpu}{core}{$cpu}{system_mode},
"s4:loads::cpu::core::${cpu}::idle_tasks" => $anvil->data->{loads}{cpu}{core}{$cpu}{idle_tasks},
"s5:loads::cpu::core::${cpu}::io_wait" => $anvil->data->{loads}{cpu}{core}{$cpu}{io_wait},
"s6:loads::cpu::core::${cpu}::hard_irq" => $anvil->data->{loads}{cpu}{core}{$cpu}{hard_irq},
"s7:loads::cpu::core::${cpu}::soft_irq" => $anvil->data->{loads}{cpu}{core}{$cpu}{soft_irq},
}});
}
elsif ($line =~ /^intr (\d+) (.*?)$/)
{
my $total_interrupts = $1;
my $other_interrupts = $2; # We might want to pull this apart later.
$anvil->data->{loads}{interrupts}{total} = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"s1:loads::interrupts::total" => $anvil->data->{loads}{interrupts}{total},
}});
}
elsif ($line =~ /^processes (\d+)$/)
{
$anvil->data->{loads}{processes}{total} = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"loads::processes::total" => $anvil->data->{loads}{processes}{total},
}});
}
elsif ($line =~ /^procs_running (\d+)$/)
{
$anvil->data->{loads}{processes}{running} = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"loads::processes::running" => $anvil->data->{loads}{processes}{running},
}});
}
elsif ($line =~ /^procs_blocked (\d+)$/)
{
$anvil->data->{loads}{processes}{blocked} = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"loads::processes::blocked" => $anvil->data->{loads}{processes}{blocked},
}});
}
}
return(0);
}
=head2 md5sum =head2 md5sum
This returns the C<< md5sum >> of a given file. This returns the C<< md5sum >> of a given file.
@ -2600,6 +2895,30 @@ sub switches
$anvil->nice_exit({exit_code => 0}); $anvil->nice_exit({exit_code => 0});
} }
# Lastly, if there's a anvil.debug file, set logging to '-vv --log-secure'
if (-e $anvil->data->{path}{configs}{'anvil.debug'})
{
# Set defaults, then see if we should override from the body.
$anvil->data->{switches}{v} = "";
$anvil->data->{switches}{v} = "";
$anvil->data->{switches}{vv} = "#!SET!#";
$anvil->data->{switches}{'log-secure'} = "#!SET!#";
$anvil->data->{defaults}{'log'}{pids} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"switches::V" => $anvil->data->{switches}{V},
"switches::v" => $anvil->data->{switches}{v},
"switches::vv" => $anvil->data->{switches}{vv},
"switches::log-secure" => $anvil->data->{switches}{'log-secure'},
"defaults::log::pids" => $anvil->data->{defaults}{'log'}{pids},
}});
### TODO: We might want to set this?
#$anvil->data->{sys}{database}{log_transactions} = 1;
# Adjust the log level if requested.
$anvil->Log->_adjust_log_level();
}
return(0); return(0);
} }

@ -35,6 +35,8 @@ my $THIS_FILE = "Network.pm";
# ping # ping
# read_nmcli # read_nmcli
# reset_connection # reset_connection
# wait_for_network
# wait_on_nm_online
# _check_firewalld_conf # _check_firewalld_conf
# _get_existing_zone_interfaces # _get_existing_zone_interfaces
# _get_server_ports # _get_server_ports
@ -4508,6 +4510,277 @@ sub reset_connection
} }
=head2 wait_for_network
This method checks for Network Manager configurations. Any that are found that belong to the Anvil![1] will be watched until their state is C<< activated >>.
B<<Note>>: 1. Interfaces with device name starting with C<< bcnX_ >>, C<< ifnX_ >>, C<< snX_ >>, or C<< mnX_ >>, where C<< X >> is an integer
B<< Note >>: This method only works on Network Manager based systems.
Parameters;
=head3 timeout (optional, default '300')
By default, this method will wait for five minutes. If you want to set a timeout, set this as a number of seconds. If the timeout expires and any bonds are still not up, the method will return C<< 1 >>. If this is set to C<< 0 >>, it will wait forever.
=cut
sub wait_for_network
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Network->wait_for_network()" }});
### TODO: all configured bonds and slaved interfaces should be 'activated' before this returns, even if their cable is out. Wait for this, with a default 300s timeout.
my $timeout = defined $parameter->{timeout} ? $parameter->{timeout} : 300;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
timeout => $timeout,
}});
# If timeout wasn't set, but network::wait_for_network::timeout is set, use it.
if ((exists $anvil->data->{network}{wait_for_network}{timeout}) && ($anvil->data->{network}{wait_for_network}{timeout} =~ /^\d+$/))
{
$timeout = $anvil->data->{network}{wait_for_network}{timeout};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { timeout => $timeout }});
}
my $short_host_name = $anvil->Get->short_host_name();
my $directory = $anvil->data->{path}{directories}{NetworkManager};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
short_host_name => $short_host_name,
directory => $directory,
}});
if (not -d $directory)
{
return(0);
}
$directory .= "/system-connections";
$directory =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { directory => $directory }});
# Find any bonds.
local(*DIRECTORY);
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0018", variables => { directory => $directory }});
opendir(DIRECTORY, $directory);
while(my $file = readdir(DIRECTORY))
{
next if $file !~ /\.nmconnection$/;
my $full_path = $directory."/".$file;
$full_path =~ s/\/\//\//g;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { full_path => $full_path }});
my $file_body = $anvil->Storage->read_file({debug => $debug, file => $full_path});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { file_body => $file_body }});
my $uuid = "";
my $type = "";
my $interface_name = "";
my $id = "";
my $parent_bond = "";
foreach my $line (split/\n/, $file_body)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
# Collect the UUID
if ($line =~ /^uuid=(.*)$/)
{
$uuid = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { uuid => $uuid }});
}
# ethernet, bond, or bridge
if ($line =~ /^type=(.*)$/)
{
$type = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { type => $type }});
}
# Get the device name
if ($line =~ /^interface-name=(.*)$/)
{
$interface_name = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
}
if ($line =~ /id=(.*)$/)
{
$id = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { id => $id }});
}
# Find the parent bond, if this is a child interface
if ($line =~ /^master=(.*)$/)
{
$parent_bond = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { parent_bond => $parent_bond }});
}
}
if ($uuid)
{
# If the interface_name is multiple names, pull our the name we use (if name)
if ((not $interface_name) && ($id))
{
$interface_name = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
}
if ($interface_name =~ /;/)
{
$interface_name =~ s/;$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
foreach my $this_interface_name (split/;/, $interface_name)
{
if (($this_interface_name =~ /^bcn\d+_/) or
($this_interface_name =~ /^ifn\d+_/) or
($this_interface_name =~ /^sn\d+_/) or
($this_interface_name =~ /^mn\d+_/))
{
$interface_name = $this_interface_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
}
}
}
next if not $interface_name;
# Is this an interface we care about?
if (($interface_name =~ /^bcn\d+_/) or
($interface_name =~ /^ifn\d+_/) or
($interface_name =~ /^sn\d+_/) or
($interface_name =~ /^mn\d+_/))
{
# Watch for this interface
$anvil->data->{network}{watch}{$interface_name}{uuid} = $uuid;
$anvil->data->{network}{watch}{$interface_name}{type} = $type;
$anvil->data->{network}{watch}{$interface_name}{ready} = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"network::watch::${interface_name}::uuid" => $anvil->data->{network}{watch}{$interface_name}{uuid},
"network::watch::${interface_name}::type" => $anvil->data->{network}{watch}{$interface_name}{type},
}});
}
}
}
closedir(DIRECTORY);
my $waiting = 1;
my $end_time = $timeout ? time + $timeout : 0;
my $duration = $end_time ? $timeout - time : 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
end_time => $end_time,
duration => $duration,
}});
while($waiting)
{
$anvil->Network->read_nmcli({
debug => $debug,
host => $short_host_name,
});
$waiting = 0;
foreach my $interface_name (sort {$a cmp $b} keys %{$anvil->data->{network}{watch}})
{
next if $anvil->data->{network}{watch}{$interface_name}{ready};
my $uuid = $anvil->data->{network}{watch}{$interface_name}{uuid};
my $type = $anvil->data->{network}{watch}{$interface_name}{type};
my $state = $anvil->data->{nmcli}{$short_host_name}{uuid}{$uuid}{'state'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:interface_name' => $interface_name,
's2:type' => $type,
's3:uuid' => $uuid,
's4:state' => $state,
}});
if (($state eq "activated") or ($state == 1))
{
$anvil->data->{network}{watch}{$interface_name}{ready} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"network::watch::${interface_name}::ready" => $anvil->data->{network}{watch}{$interface_name}{ready},
}});
}
}
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { waiting => $waiting }});
if ($waiting)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { end_time => $end_time }});
if ($end_time)
{
if (time > $end_time)
{
# We're done.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0847", variables => { timeout => $timeout }});
return(1);
}
my $time_left = $end_time - time;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { time_left => $time_left }});
}
# Sleep for a few seconds.
sleep 5;
}
}
return(0);
}
=head2 wait_on_nm_online
This method calls C<< nm-online --wait-for-startup --timeout X >>, which in turn waits for Network Manager to report C<< startup complete >> in the journald logs. The default timeout used here is C<< 120 >> seconds (as opposed to the default of C<< 30 >> used by C<< nm-online >> itself).
From our testing, given the complexity of the network in Anvil! clusters, this much time isn't out of the ordinaryl
Feb 24 19:13:17 an-a01n01.ci.alteeve.com NetworkManager[1003]: <info> [1708801997.5155] NetworkManager (version 1.44.0-4.el9_3) is starting... (boot:833ea5be-eb44-4214-9e2d-8c6281dec9b6)
...
Feb 24 19:14:53 an-a01n01.ci.alteeve.com NetworkManager[1003]: <info> [1708802093.9684] manager: startup complete
B<< Note >>: This method only works on Network Manager based systems.
The return code from C<< nm-online >> is returned. See C<< man nm-online >> for details, but the main return codes are C<< 0 >> meaning the connection came up within the timeout, C<< 1 >> if the connection failed to come up within the timeout, and C<< 2 >> if there was any error.
Parameters;
=head3 timeout (optional, default '120')
By default, this method will wait for two minutes. If you want to set a timeout, set this as a number of seconds. If the timeout expires and any bonds are still not up, the method will return C<< 1 >>. If this is set to C<< 0 >>, it will wait forever.
=cut
sub wait_on_nm_online
{
my $self = shift;
my $parameter = shift;
my $anvil = $self->parent;
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Network->wait_on_nm_online()" }});
my $timeout = defined $parameter->{timeout} ? $parameter->{timeout} : 120;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
timeout => $timeout,
}});
if ((not $timeout) or ($timeout !~ /^\d+$/))
{
# Invalid timeout.
$timeout = 120;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { timeout => $timeout }});
}
my $shell_call = $anvil->data->{path}{exe}{'nm-online'}." --wait-for-startup --quiet --timeout ".$timeout;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
's1:output' => $output,
's2:return_code' => $return_code,
}});
return($return_code);
}
############################################################################################################# #############################################################################################################
# Private functions # # Private functions #
############################################################################################################# #############################################################################################################
@ -4706,7 +4979,7 @@ sub _get_drbd_ports
# DRBD isn't installed. # DRBD isn't installed.
return(0); return(0);
} }
local(*DIRECTORY); local(*DIRECTORY);
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0018", variables => { directory => $directory }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0018", variables => { directory => $directory }});
opendir(DIRECTORY, $directory); opendir(DIRECTORY, $directory);

@ -338,6 +338,21 @@ sub call
password => $anvil->Log->is_secure($password), password => $anvil->Log->is_secure($password),
}}); }});
} }
# Is the global "always reconnect" is set, set 'close' to 1 and clear any cached connections.
$anvil->data->{sys}{net}{always_reconnect} = 0 if not defined $anvil->data->{sys}{net}{always_reconnect};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
"sys::net::always_reconnect" => $anvil->data->{sys}{net}{always_reconnect},
}});
if ($anvil->data->{sys}{net}{always_reconnect})
{
$close = 1;
$no_cache = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
'close' => $close,
no_cache => $no_cache,
}});
}
### NOTE: This caused problems that are currently unsolved. ### NOTE: This caused problems that are currently unsolved.
=cut =cut
@ -495,6 +510,7 @@ sub call
's3:remote_user' => $remote_user, 's3:remote_user' => $remote_user,
's4:port' => $port, 's4:port' => $port,
}}); }});
alarm(120);
($connect_output) = capture_merged { ($connect_output) = capture_merged {
$ssh_fh = Net::OpenSSH->new($target, $ssh_fh = Net::OpenSSH->new($target,
user => $remote_user, user => $remote_user,
@ -510,6 +526,7 @@ sub call
's2:ssh_fh->error' => $ssh_fh->error, 's2:ssh_fh->error' => $ssh_fh->error,
's3:connect_output' => $connect_output, 's3:connect_output' => $connect_output,
}}); }});
alarm(0);
# Any fatal issues reaching the target? # Any fatal issues reaching the target?
if ($connect_output =~ /Could not resolve hostname/i) if ($connect_output =~ /Could not resolve hostname/i)
@ -688,10 +705,12 @@ sub call
$error = ""; $error = "";
if ($timeout) if ($timeout)
{ {
# Call with a timeout # Call with a timeout. Use alarm also, as capture2's timeout is questionaly reliable.
alarm($timeout + 60);
($output, $error) = $ssh_fh->capture2({timeout => $timeout}, $shell_call); ($output, $error) = $ssh_fh->capture2({timeout => $timeout}, $shell_call);
$output = "" if not defined $output; $output = "" if not defined $output;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { 'ssh_fh->error' => $ssh_fh->error }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => $secure, list => { 'ssh_fh->error' => $ssh_fh->error }});
alarm(0);
} }
else else
{ {

@ -2721,15 +2721,27 @@ LIMIT 1;";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0673", variables => { host_name => $host_name }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0673", variables => { host_name => $host_name }});
$shell_call =~ s/--action status/ --action on/; $shell_call =~ s/--action status/ --action on/;
my ($output, $return_code) = $anvil->System->call({debug => $debug, timeout => 30, shell_call => $shell_call}); my ($output, $return_code) = $anvil->System->call({debug => 1, timeout => 30, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
# Mark it as booting. if ($return_code)
$anvil->Database->update_host_status({ {
debug => $debug, # Failed to boot.
host_uuid => $host_uuid, $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0170", variables => {
host_status => "booting", host_name => $host_name,
}); return_code => $return_code,
output => $output,
}});
}
else
{
# Mark it as booting.
$anvil->Database->update_host_status({
debug => $debug,
host_uuid => $host_uuid,
host_status => "booting",
});
}
} }
} }
} }

@ -4202,7 +4202,7 @@ sub push_file
=head2 read_config =head2 read_config
This method is used to read 'Anvil::Tools' style configuration files. These configuration files are in the format: This method is used to read C<< Anvil::Tools >> style configuration files. These configuration files are in the format:
# This is a comment for the 'a::b::c' variable # This is a comment for the 'a::b::c' variable
a::b::c = x a::b::c = x
@ -5228,6 +5228,10 @@ B<< Note >>: If the variable is not found, it is treated like an error and C<< 1
Parameters; Parameters;
=head3 append (optional, default 0)
If set to C<< 1 >>, and if the variable is not found, it will be appended to the end of the config file.
=head3 port (optional, default 22) =head3 port (optional, default 22)
If C<< target >> is set, this is the TCP port number used to connect to the remote machine. If C<< target >> is set, this is the TCP port number used to connect to the remote machine.
@ -5267,6 +5271,7 @@ sub update_config
my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->update_config()" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->update_config()" }});
my $append = defined $parameter->{append} ? $parameter->{append} : 0;
my $password = defined $parameter->{password} ? $parameter->{password} : ""; my $password = defined $parameter->{password} ? $parameter->{password} : "";
my $port = defined $parameter->{port} ? $parameter->{port} : 22; my $port = defined $parameter->{port} ? $parameter->{port} : 22;
my $secure = defined $parameter->{secure} ? $parameter->{secure} : ""; my $secure = defined $parameter->{secure} ? $parameter->{secure} : "";
@ -5278,6 +5283,7 @@ sub update_config
my $update = 0; my $update = 0;
my $new_file = ""; my $new_file = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 0, list => {
append => $append,
password => $anvil->Log->is_secure($password), password => $anvil->Log->is_secure($password),
port => $port, port => $port,
secure => $secure, secure => $secure,
@ -5343,22 +5349,35 @@ sub update_config
# Did we see the variable? # Did we see the variable?
if (not $seen) if (not $seen)
{ {
if ($anvil->Network->is_local({host => $target})) if ($append)
{ {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0174", variables => { # Add the variable to the config file.
variable => $variable, $new_file .= $variable."\t=\t".$value."\n";
file => $anvil->data->{path}{configs}{'anvil.conf'}, $update = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, secure => 1, list => {
new_file => $new_file,
update => $update,
}}); }});
return(1);
} }
else else
{ {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0175", variables => { if ($anvil->Network->is_local({host => $target}))
variable => $variable, {
file => $anvil->data->{path}{configs}{'anvil.conf'}, $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0174", variables => {
target => $target, variable => $variable,
}}); file => $anvil->data->{path}{configs}{'anvil.conf'},
return(1); }});
return(1);
}
else
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0175", variables => {
variable => $variable,
file => $anvil->data->{path}{configs}{'anvil.conf'},
target => $target,
}});
return(1);
}
} }
} }

@ -2936,7 +2936,7 @@ sub generate_state_json
{ {
my $type = $anvil->data->{network}{$host}{interface}{$interface}{type}; my $type = $anvil->data->{network}{$host}{interface}{$interface}{type};
my $uuid = $anvil->data->{network}{$host}{interface}{$interface}{uuid}; my $uuid = $anvil->data->{network}{$host}{interface}{$interface}{uuid};
my $mtu = $anvil->data->{network}{$host}{interface}{$interface}{mtu}; my $mtu = $anvil->data->{network}{$host}{interface}{$interface}{mtu} ? $anvil->data->{network}{$host}{interface}{$interface}{mtu} : 1500;
my $mac_address = $anvil->data->{network}{$host}{interface}{$interface}{mac_address}; my $mac_address = $anvil->data->{network}{$host}{interface}{$interface}{mac_address};
my $iface_hash = {}; my $iface_hash = {};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
@ -2996,6 +2996,22 @@ sub generate_state_json
} }
elsif ($type eq "bond") elsif ($type eq "bond")
{ {
if ((not exists $anvil->data->{network}{$host}) or
(not exists $anvil->data->{network}{$host}{interface}{$interface}) or
(not defined $anvil->data->{network}{$host}{interface}{$interface}{mode}))
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
"s1:host" => $host,
"s2:interface" => $interface,
"s3:mac_address" => $mac_address,
"s4:type" => $type,
"s5:mtu" => $mtu,
"s6:configured" => $configured,
"s7:host_uuid" => $host_uuid,
"s8:host_key" => $host_key,
}});
next;
}
my $mode = $anvil->data->{network}{$host}{interface}{$interface}{mode}; my $mode = $anvil->data->{network}{$host}{interface}{$interface}{mode};
my $primary_interface = $anvil->data->{network}{$host}{interface}{$interface}{primary_interface}; my $primary_interface = $anvil->data->{network}{$host}{interface}{$interface}{primary_interface};
my $primary_reselect = $anvil->data->{network}{$host}{interface}{$interface}{primary_reselect}; my $primary_reselect = $anvil->data->{network}{$host}{interface}{$interface}{primary_reselect};
@ -5249,13 +5265,13 @@ sub update_hosts
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { line => $line }});
if ($line) if ($line)
{ {
$last_line_blank = 0; $last_line_blank = 0;
$cleaned_body .= $line."\n"; $cleaned_body .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_line_blank => $last_line_blank }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_line_blank => $last_line_blank }});
} }
elsif (not $last_line_blank) elsif (not $last_line_blank)
{ {
$last_line_blank = 1; $last_line_blank = 1;
$cleaned_body .= $line."\n"; $cleaned_body .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_line_blank => $last_line_blank }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { last_line_blank => $last_line_blank }});
} }
@ -5474,6 +5490,41 @@ sub update_hosts
}}); }});
} }
# Clean spaces off the end of lines, and look for invalid entries.
$cleaned_body = "";
foreach my $line (split/\n/, $new_body)
{
$line =~ s/\s+$//;
$line =~ s/^\s+(\d.*)$/$1/;
if ($line =~ /^#/)
{
$cleaned_body .= $line."\n";
}
elsif ($line =~ /^(.*?)\s+(.*?)/)
{
my $ip = $1;
my $name = $2;
if ($anvil->Validate->ip({ip => $ip}))
{
# Valid
$cleaned_body .= $line."\n";
}
else
{
# The is not a valid hosts entry.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "warning_0172", variables => { line => $line }});
}
}
}
$difference = "";
$difference = diff \$new_body, \$cleaned_body, { STYLE => 'Unified' };
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { difference => $difference }});
if ($difference)
{
$new_body = $cleaned_body;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { new_body => $new_body }});
}
my $new_line_count = @{$ip_order}; my $new_line_count = @{$ip_order};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { new_line_count => $new_line_count }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { new_line_count => $new_line_count }});
if ($new_line_count) if ($new_line_count)
@ -5482,6 +5533,10 @@ sub update_hosts
foreach my $ip_address (@{$ip_order}) foreach my $ip_address (@{$ip_order})
{ {
# Clean off trailing white spaces.
$lines->{$ip_address} =~ s/\s+$//;
# Add it.
$new_body .= $lines->{$ip_address}."\n"; $new_body .= $lines->{$ip_address}."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { new_body => $new_body }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { new_body => $new_body }});
} }

@ -224,3 +224,9 @@ sys::manage::firewall = 1
# behavious by setting this to '0'. Be aware of course that you will need to manually update or add entries # behavious by setting this to '0'. Be aware of course that you will need to manually update or add entries
# going forward. # going forward.
#sys::hosts::manage = 0 #sys::hosts::manage = 0
# During the startup of a machine, Network Manager can take a minute or two before the network is stable. By
# default, anvil-daemon and scancore will pause during startup, waiting for the network to stabilize. By
# default, Network->wait_for_network waits for five minutes. During the initial config, we keep this shorter.
# This value will be updated by anvil-configure-host when the initial configuration completes.
network::wait_for_network::timeout = 300

@ -56,6 +56,7 @@ Requires: glibc-all-langpacks
Requires: gpm Requires: gpm
Requires: hdparm Requires: hdparm
Requires: htop Requires: htop
Requires: iotop
Requires: iproute Requires: iproute
Requires: kernel-core Requires: kernel-core
Requires: kernel-devel Requires: kernel-devel
@ -106,6 +107,7 @@ Requires: screen
Requires: smartmontools Requires: smartmontools
Requires: strace Requires: strace
Requires: syslinux Requires: syslinux
Requires: sysstat
Requires: tar Requires: tar
Requires: tcpdump Requires: tcpdump
Requires: tmux Requires: tmux
@ -247,7 +249,9 @@ setenforce 0
### TODO: check it if was disabled (if it existed before) and, if so, leave it disabled. ### TODO: check it if was disabled (if it existed before) and, if so, leave it disabled.
systemctl enable --now chronyd.service systemctl enable --now chronyd.service
systemctl enable --now anvil-daemon.service systemctl enable --now anvil-daemon.service
systemctl enable --now anvil-monitor-daemons.service
systemctl enable --now anvil-monitor-network.service systemctl enable --now anvil-monitor-network.service
systemctl enable --now anvil-monitor-performance.service
systemctl enable --now scancore.service systemctl enable --now scancore.service
%pre striker %pre striker

@ -20,6 +20,7 @@ dist_man8_MANS = \
anvil-join-anvil.8 \ anvil-join-anvil.8 \
anvil-maintenance-mode.8 \ anvil-maintenance-mode.8 \
anvil-manage-alerts.8 \ anvil-manage-alerts.8 \
anvil-manage-daemons.8 \
anvil-manage-dr.8 \ anvil-manage-dr.8 \
anvil-manage-files.8 \ anvil-manage-files.8 \
anvil-manage-firewall.8 \ anvil-manage-firewall.8 \
@ -28,6 +29,8 @@ dist_man8_MANS = \
anvil-manage-server.8 \ anvil-manage-server.8 \
anvil-manage-server-storage.8 \ anvil-manage-server-storage.8 \
anvil-manage-storage-groups.8 \ anvil-manage-storage-groups.8 \
anvil-monitor-network.8 \
anvil-monitor-performance.8 \
anvil-migrate-server.8 \ anvil-migrate-server.8 \
anvil-network-profiler.8 \ anvil-network-profiler.8 \
anvil-parse-fence-agents.8 \ anvil-parse-fence-agents.8 \

@ -16,7 +16,7 @@ This method, when used with '\fB\-\-server\fR all', will honour server boot prio
\-?, \-h, \fB\-\-help\fR \-?, \-h, \fB\-\-help\fR
Show this man page. Show this man page.
.TP .TP
\fB\-\-log-secure\fR \fB\-\-log\-secure\fR
When logging, record sensitive data, like passwords. When logging, record sensitive data, like passwords.
.TP .TP
\-v, \-vv, \-vvv \-v, \-vv, \-vvv

@ -0,0 +1,51 @@
.\" Manpage for the Anvil! daemon managers
.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
.TH anvil-manage-daemons "8" "August 02 2022" "Anvil! Intelligent Availability™ Platform"
.SH NAME
anvil-manage-daemons \- Tool used to monitor and manage Anvil! daemons.
.SH SYNOPSIS
.B anvil-manage-daemons
\fI\,<command> \/\fR[\fI\,options\/\fR]
.SH DESCRIPTION
When run with '\fB\-\-monitor\fR', it will run as a daemon, checking all other Anvil! daemons. If any are found to be 'failed', they will be stopped and restarted.
.TP
.SH OPTIONS
.TP
\-?, \-h, \fB\-\-help\fR
Show this man page.
.TP
\fB\-\-log\-secure\fR
When logging, record sensitive data, like passwords.
.TP
\-v, \-vv, \-vvv
Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
.SS "Commands:"
.TP
\fB\-\-enable\fR
All Anvil! daemons that are not enabled will be enabled.
.TP
\fB\-\-disable\fR
All Anvil! daemons that are not disabled will be disabled.
.TP
\fB\-\-log\-only\fR
if this is passed, it monitors the status of the daemons, but does not try to recover failed ones.
.TP
\fB\-\-monitor\fR
This is set to the job UUID when the request to boot is coming from a database job. When set, the referenced job will be updated and marked as complete / failed when the run completes.
.TP
\fB\-\-now\fR
This can be used with \fB\-\-enable\fR or \fB\-\-disable\fR to have the daemons started or stopped immediately.
.TP
\fB\-\-start\fR
This will start all daemons that are not already running.
.TP
\fB\-\-status\fR
Reports the state of all daemons.
.TP
\fB\-\-stop\fR
This will stop all daemons that are not already stopped.
.IP
.SH AUTHOR
Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
.SH "REPORTING BUGS"
Report bugs to users@clusterlabs.org

@ -0,0 +1,41 @@
.\" Manpage for the Anvil! server boot program
.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
.TH anvil-monitor-performance "8" "February 12 2024" "Anvil! Intelligent Availability™ Platform"
.SH NAME
anvil-monitor-performance \- Tool used to log system performance to anvil.conf to assist with performance issue debugging.
.SH SYNOPSIS
.B anvil-monitor-performance
\fI\,<command> \/\fR[\fI\,options\/\fR]
.SH DESCRIPTION
This tool uses the Get->load_average() method to collect performance data and then records the parsed data to the Anvil! logs.
.TP
This is meant to be a light-weight tool run by the same-named systemd daemon. It does not connect to the database.
.TP
.SH OPTIONS
.TP
\-?, \-h, \fB\-\-help\fR
Show this man page.
.TP
\fB\-\-log\-secure\fR
When logging, record sensitive data, like passwords.
.TP
\-v, \-vv, \-vvv
Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
.SS "Commands:"
.TP
\fB\-\-detailed\fR
Log extended performance data. This shows extended and per-cpu-core and per block device metrics.
.TP
\fB\-\-interval\fR <seconds>
By default, the performance data is collected and logged every five seconds. If you want to change this frequency, you can use this switch to set the interval seconds you wish to use.
.TP
\fB\-\-print\fR
By default, this programs logs to /var/log/anvil.log. This switch also sends the logged data to STDOUT.
.TP
\fB\-\-run\-once\fR <uuid>
This tells the program to collect and report the performance data once, and then to exit.
.IP
.SH AUTHOR
Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
.SH "REPORTING BUGS"
Report bugs to users@clusterlabs.org

@ -0,0 +1,30 @@
.\" Manpage for the Anvil! Striker dashboard screenshot collection tool
.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions.
.TH striker-get-screenshots "8" "September 12 2023" "Anvil! Intelligent Availability™ Platform"
.SH NAME
striker-get-screenshots \- Tool used to collect screenshots from Anvil! subnodes and DR hosts
.SH SYNOPSIS
.B striker-get-screenshots
\fI\,<command> \/\fR[\fI\,options\/\fR]
.SH DESCRIPTION
striker-get-screenshots \- This uses the libvirtd API to collect screenshots from servers running on Anvil! subnodes and DR hosts. These are used to display periodic views into servers on the Striker dashboard.
.TP
.SH OPTIONS
.TP
\-?, \-h, \fB\-\-help\fR
Show this man page.
.TP
\fB\-\-log-secure\fR
When logging, record sensitive data, like passwords.
.TP
\-v, \-vv, \-vvv
Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data.
.SS "Commands:"
.TP
\fB\-\-job\-uuid\fR
This is the jobs -> job_uuid to execute. Generally this is only used by other programs.
.IP
.SH AUTHOR
Written by Madison Kelly, Alteeve staff and the Anvil! project contributors.
.SH "REPORTING BUGS"
Report bugs to users@clusterlabs.org

@ -248,6 +248,18 @@ elsif ((not $anvil->data->{switches}{server}) && ($anvil->data->{environment}{OC
}}); }});
} }
# In some cases, 'OCF_RESKEY_name' isn't set, but 'OCF_RESOURCE_INSTANCE' is. If this is the case, copy the
# data over.
if ((not $anvil->data->{environment}{OCF_RESKEY_name}) && ($anvil->data->{environment}{OCF_RESOURCE_INSTANCE}))
{
$anvil->data->{environment}{OCF_RESKEY_name} = $anvil->data->{environment}{OCF_RESOURCE_INSTANCE};
$anvil->data->{switches}{server} = $anvil->data->{environment}{OCF_RESOURCE_INSTANCE};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"environment::OCF_RESKEY_name" => $anvil->data->{environment}{OCF_RESKEY_name},
"switches::server" => $anvil->data->{switches}{server},
}});
}
if (($anvil->data->{switches}{migrate_to}) && (not $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target})) if (($anvil->data->{switches}{migrate_to}) && (not $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target}))
{ {
$anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = $anvil->data->{switches}{migrate_to}; $anvil->data->{environment}{OCF_RESKEY_CRM_meta_migrate_target} = $anvil->data->{switches}{migrate_to};

@ -1761,6 +1761,7 @@ Note: This is a permanent action! If you protect this server again later, a full
<key name="job_0478">Preparing the new definition file</key> <key name="job_0478">Preparing the new definition file</key>
<key name="job_0479">The new definition is ready, saving it.</key> <key name="job_0479">The new definition is ready, saving it.</key>
<key name="job_0480">Running as a job, not prompting the user to confirm.</key> <key name="job_0480">Running as a job, not prompting the user to confirm.</key>
<key name="job_0481">The fence device: [#!variable!device!#] already exists.</key>
<!-- Log entries --> <!-- Log entries -->
<key name="log_0001">Starting: [#!variable!program!#].</key> <key name="log_0001">Starting: [#!variable!program!#].</key>
@ -2714,7 +2715,19 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
<key name="log_0832">The host: [#!variable!host_name!#] was found not found in the '/etc/hosts' file! We'll wait a few seconds and check again.</key> <key name="log_0832">The host: [#!variable!host_name!#] was found not found in the '/etc/hosts' file! We'll wait a few seconds and check again.</key>
<key name="log_0833">All host names were found in '/etc/hosts', ready to proceed!</key> <key name="log_0833">All host names were found in '/etc/hosts', ready to proceed!</key>
<key name="log_0834">One or more hosts are not yet in the '/etc/hosts' file with expected IPs. We'll wait a short bit and check again.</key> <key name="log_0834">One or more hosts are not yet in the '/etc/hosts' file with expected IPs. We'll wait a short bit and check again.</key>
<key name="log_0835">There are entries we need to add to the '/etc/hosts' file before we can form the cluster, updating it now.</key> <key name="log_0835">The CPU load average is; (one / five / ten minutes): [#!variable!one_minute!# / #!variable!five_minutes!# / #!variable!ten_minutes!#].</key>
<key name="log_0836">Processes; (total, running, blocked): [#!variable!total!#, #!variable!running!#, #!variable!blocked!#]</key>
<key name="log_0837">Time spend (in secs) doing; IO wait: [#!variable!io_wait!#], user mode: [#!variable!user_mode!#], niced user mode: [#!variable!user_mode_nice!#], system mode: [#!variable!system_mode!#], idle tasks: [#!variable!idle_tasks!#], hard IRQ: [#!variable!hard_irq!#], soft IRQ: [#!variable!soft_irq!#].</key>
<key name="log_0838">CPU Core: [#!variable!core!#], time doind (seconds); user mode: [#!variable!user_mode!#], niced user mode: [#!variable!user_mode_nice!#], system mode: [#!variable!system_mode!#], idle tasks: [#!variable!idle_tasks!#], hard IRQ: [#!variable!hard_irq!#], soft IRQ: [#!variable!soft_irq!#].</key>
<key name="log_0839">Drive: [#!variable!device_name!#], IOs currently in progress: [#!variable!in_progress!#], weighted time spent: [#!variable!weighted_time_spent!# sec].</key>
<key name="log_0840">CPU percent time doing; (IO wait / user / system / nice'd / idle / involuntary wait): [#!variable!iowait!# / #!variable!user!# / #!variable!system!# / #!variable!nice!# / #!variable!idle!#/ #!variable!steal!#].</key>
<key name="log_0841">Processes; (total / running / blocked / IRQ interrupts): [#!variable!total!# / #!variable!running!# / #!variable!blocked!# / #!variable!interrupts!#].</key>
<key name="log_0842">Starting the network manager connection: [#!variable!id!#] (uuid: [#!variable!uuid!#]).</key>
<key name="log_0843">Creating the: [#!variable!file!#] debug file on: [#!variable!host!#].</key>
<key name="log_0844">The bond interface: [#!variable!bond_name!#] is not ready yet, we'll check again shortly.</key>
<key name="log_0845">No interfaces found configured for the bond: [#!variable!bond_name!#], ignoring it.</key>
<key name="log_0846">CPU Load Average: [#!variable!one_minute!# / #!variable!five_minutes!# / #!variable!ten_minutes!#], io wait: [#!variable!iowait!#%], running/blocked processes: [#!variable!running!# / #!variable!blocked!#]</key>
<key name="log_0847">Times out waiting for the bonds to come up. We waited: [#!variable!waited!#] seconds.</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. --> <!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key> <key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>
@ -3354,6 +3367,9 @@ proceeding.
<key name="message_0416">[ Note ] - The network has reconnected to the database, configuring will complete shortly.</key> <key name="message_0416">[ Note ] - The network has reconnected to the database, configuring will complete shortly.</key>
<key name="message_0417">[ Note ] - The old 'ifcfg' style config file: [#!variable!file!#] will be backed up and then removed!</key> <key name="message_0417">[ Note ] - The old 'ifcfg' style config file: [#!variable!file!#] will be backed up and then removed!</key>
<key name="message_0418">[ Note ] - Updated the ssh daemon config file: [#!variable!file!#] to enable ssh access for the root user.</key> <key name="message_0418">[ Note ] - Updated the ssh daemon config file: [#!variable!file!#] to enable ssh access for the root user.</key>
<key name="message_0419">Anvil! Intelligent Availability Daemon Status (Enabled, Started, Failed);</key>
<key name="message_0420">- #!variable!daemon!# #!variable!enabled!# (#!variable!enable_string!#), #!variable!started!# (#!variable!start_string!#), #!variable!failed!# (#!variable!fail_string!#).</key>
<key name="message_0421">Daemons [enabled/started/failed]; #!variable!string!#.</key>
<!-- Translate names (protocols, etc) --> <!-- Translate names (protocols, etc) -->
<key name="name_0001">Normal Password</key> <!-- none in mail-server --> <key name="name_0001">Normal Password</key> <!-- none in mail-server -->
@ -4165,6 +4181,9 @@ We will try to proceed anyway.</key>
</key> </key>
<key name="warning_0168">Please specify a storage group to use to add the new drive to.</key> <key name="warning_0168">Please specify a storage group to use to add the new drive to.</key>
<key name="warning_0169">[ Warning ] - After reconfiguring the network, we've failed to connect to any database for two minutes. Rebooting in case this fixes the connection.</key> <key name="warning_0169">[ Warning ] - After reconfiguring the network, we've failed to connect to any database for two minutes. Rebooting in case this fixes the connection.</key>
<key name="warning_0170">[ Warning ] - The attempt to boot: [#!variable!host_name!#] appears to have failed. The return code received was: [#!variable!return_code!#] (expected '0'). The output, if any, was: [#!variable!output!#].</key>
<key name="warning_0171">[ Warning ] - The daemon: [#!variable!daemon!#] appears to have failed! Attempting to restart it now.</key>
<key name="warning_0172">[ Warning ] - The line: [#!variable!line!#] that was going to be added to the hosts file is invalid, removing it.</key>
</language> </language>
<!-- 日本語 --> <!-- 日本語 -->

@ -15,6 +15,7 @@ dist_sbin_SCRIPTS = \
anvil-join-anvil \ anvil-join-anvil \
anvil-maintenance-mode \ anvil-maintenance-mode \
anvil-manage-alerts \ anvil-manage-alerts \
anvil-manage-daemons \
anvil-manage-dr \ anvil-manage-dr \
anvil-manage-files \ anvil-manage-files \
anvil-manage-firewall \ anvil-manage-firewall \
@ -27,6 +28,7 @@ dist_sbin_SCRIPTS = \
anvil-manage-vnc-pipe \ anvil-manage-vnc-pipe \
anvil-migrate-server \ anvil-migrate-server \
anvil-monitor-network \ anvil-monitor-network \
anvil-monitor-performance \
anvil-network-profiler \ anvil-network-profiler \
anvil-parse-fence-agents \ anvil-parse-fence-agents \
anvil-pcs-wrapper \ anvil-pcs-wrapper \

@ -2,6 +2,8 @@
# #
# This reports the total memory used by all processes with to passed-in program name. # This reports the total memory used by all processes with to passed-in program name.
# #
# TODO: Switch to 'smes' and see: https://superuser.com/questions/150117/how-to-get-parent-pid-of-a-given-process-in-gnu-linux-from-command-line
#
# The size in bytes is returned. If '0' is reported, check the exit code to see why. # The size in bytes is returned. If '0' is reported, check the exit code to see why.
# #
# This software was created by Alteeve's Niche! Inc. and has been released under the terms of the GNU GPL # This software was created by Alteeve's Niche! Inc. and has been released under the terms of the GNU GPL

@ -37,11 +37,27 @@ $| = 1;
my $anvil = Anvil::Tools->new(); my $anvil = Anvil::Tools->new();
# Read switches # Read switches
$anvil->Get->switches({list => ["job-uuid"], man => $THIS_FILE}); $anvil->Get->switches({list => [
"job-uuid",
"debug",
], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->level({set => 2}); if (($anvil->data->{switches}{debug}) && (not -e $anvil->data->{path}{configs}{'anvil.debug'}))
$anvil->Log->secure({set => 1}); {
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0843", variables => {
file => $anvil->data->{path}{configs}{'anvil.debug'},
host => $anvil->Get->short_host_name(),
}});
my $problem = $anvil->Storage->write_file({
file => $anvil->data->{path}{configs}{'anvil.debug'},
body => "",
overwrite => 1,
user => "root",
group => "root",
mode => "0666",
});
}
# Make sure we're running as 'root' # Make sure we're running as 'root'
# $< == real UID, $> == effective UID # $< == real UID, $> == effective UID
@ -52,8 +68,21 @@ if (($< != 0) && ($> != 0))
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }
# Make sure that we set the network->wait_on_network() timeout to 60 seconds.
$anvil->Storage->update_config({
append => 1,
variable => "network::wait_on_network::timeout",
value => 60,
});
# Because we're working with the network config, we can't reliable use cached connections.
$anvil->data->{sys}{net}{always_reconnect} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, secure => 1, list => {
"sys::net::always_reconnect" => $anvil->data->{sys}{net}{always_reconnect},
}});
# Connect # Connect
$anvil->Database->connect({debug => 2, check_for_resync => 1}); $anvil->Database->connect({check_for_resync => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0031"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "message_0031"});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
@ -74,7 +103,6 @@ reconfigure_network($anvil);
# Record that we've configured this machine. # Record that we've configured this machine.
$anvil->Database->insert_or_update_variables({ $anvil->Database->insert_or_update_variables({
debug => 2,
variable_name => "system::configured", variable_name => "system::configured",
variable_value => 1, variable_value => 1,
variable_default => "", variable_default => "",
@ -91,8 +119,15 @@ $anvil->Job->update_progress({
job_uuid => $anvil->data->{job}{uuid}, job_uuid => $anvil->data->{job}{uuid},
}); });
# Make sure that we set the network->wait_on_network() timeout back to 180.
$anvil->Storage->update_config({
append => 1,
variable => "network::wait_on_network::timeout",
value => 180,
});
# Clear maintenance mode. # Clear maintenance mode.
$anvil->System->maintenance_mode({set => 0, debug => 2}); $anvil->System->maintenance_mode({set => 0});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0467"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, 'print' => 1, key => "log_0467"});
$anvil->nice_exit({exit_code => 0}); $anvil->nice_exit({exit_code => 0});
@ -210,12 +245,8 @@ sub update_passwords
my $shell_call = $anvil->data->{path}{exe}{'anvil-change-password'}." -y --password-file ".$temp_file.$anvil->Log->switches; my $shell_call = $anvil->data->{path}{exe}{'anvil-change-password'}." -y --password-file ".$temp_file.$anvil->Log->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 2, timeout => 15, shell_call => $shell_call }); my ($output, $return_code) = $anvil->System->call({timeout => 15, shell_call => $shell_call });
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { output => $output, return_code => $return_code }});
foreach my $line (split/\n/, $output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
}
if ($return_code) if ($return_code)
{ {
@ -314,7 +345,7 @@ sub reconfigure_network
} }
# Set the host_name # Set the host_name
my ($host_name, $descriptive_host_name) = $anvil->System->host_name({set => $new_host_name, pretty => $pretty_host_name, debug => 3}); my ($host_name, $descriptive_host_name) = $anvil->System->host_name({set => $new_host_name, pretty => $pretty_host_name});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
host_name => $host_name, host_name => $host_name,
descriptive_host_name => $descriptive_host_name, descriptive_host_name => $descriptive_host_name,
@ -476,7 +507,7 @@ ORDER BY
$anvil->data->{sys}{reboot} = 0; $anvil->data->{sys}{reboot} = 0;
# Read the existing network data # Read the existing network data
$anvil->Network->collect_data({debug => 2, start => 1}); $anvil->Network->collect_data({start => 1});
# This will be set to '1' if we make a change. # This will be set to '1' if we make a change.
my $changes = 0; my $changes = 0;
@ -503,7 +534,7 @@ ORDER BY
file => $THIS_FILE, file => $THIS_FILE,
line => __LINE__, line => __LINE__,
}); });
$anvil->Database->disconnect({debug => 2}); $anvil->Database->disconnect();
} }
# These can brake the connection. # These can brake the connection.
@ -524,7 +555,7 @@ ORDER BY
while ($waiting) while ($waiting)
{ {
$anvil->refresh(); $anvil->refresh();
$anvil->Database->connect({debug => 2}); $anvil->Database->connect();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections} }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections} }});
if ($anvil->data->{sys}{database}{connections}) if ($anvil->data->{sys}{database}{connections})
@ -568,8 +599,11 @@ ORDER BY
} }
# If any virtio bridges exist, remove it/them. # If any virtio bridges exist, remove it/them.
my $start = 0; my $start = 0;
my ($bridges, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-list"}); my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-list";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($bridges, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridges => $bridges, return_code => $return_code }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridges => $bridges, return_code => $return_code }});
if ($return_code) if ($return_code)
{ {
@ -726,7 +760,7 @@ sub reconfigure_bridges
else else
{ {
# If there are ifcfg files for this bridge, move it. # If there are ifcfg files for this bridge, move it.
my $network_type = $anvil->System->check_network_type({debug => 2}); my $network_type = $anvil->System->check_network_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }});
if ($network_type eq "ifcfg") if ($network_type eq "ifcfg")
{ {
@ -745,7 +779,7 @@ sub reconfigure_bridges
job_uuid => $anvil->data->{job}{uuid}, job_uuid => $anvil->data->{job}{uuid},
variables => { file => $ifcfg_file }, variables => { file => $ifcfg_file },
}); });
$anvil->Storage->backup({debug => 2, file => $ifcfg_file}); $anvil->Storage->backup({file => $ifcfg_file});
unlink $ifcfg_file; unlink $ifcfg_file;
} }
} }
@ -797,8 +831,8 @@ sub reconfigure_bridges
device_uuid => $bridge_uuid, device_uuid => $bridge_uuid,
}, },
}); });
my ($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $bridge_uuid, variable => "ipv4.method", value => "disabled"}); my ($output, $return_code) = $anvil->Network->modify_connection({uuid => $bridge_uuid, variable => "ipv4.method", value => "disabled"});
($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $bridge_uuid, variable => "ipv6.method", value => "disabled"}); ($output, $return_code) = $anvil->Network->modify_connection({uuid => $bridge_uuid, variable => "ipv6.method", value => "disabled"});
my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$bridge_name; my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$bridge_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@ -820,7 +854,7 @@ sub reconfigure_bridges
line => __LINE__, line => __LINE__,
variables => { }, variables => { },
}); });
$anvil->Network->collect_data({debug => 2}); $anvil->Network->collect_data();
} }
# Checking that the device is connected to this bridge # Checking that the device is connected to this bridge
@ -906,8 +940,8 @@ sub reconfigure_bridges
device_uuid => $on_device_uuid, device_uuid => $on_device_uuid,
}, },
}); });
my ($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $on_device_uuid, variable => "ipv4.method", value => "disabled"}); my ($output, $return_code) = $anvil->Network->modify_connection({uuid => $on_device_uuid, variable => "ipv4.method", value => "disabled"});
($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $on_device_uuid, variable => "ipv6.method", value => "disabled"}); ($output, $return_code) = $anvil->Network->modify_connection({uuid => $on_device_uuid, variable => "ipv6.method", value => "disabled"});
# Connect it now. # Connect it now.
$anvil->Job->update_progress({ $anvil->Job->update_progress({
@ -962,7 +996,7 @@ sub reconfigure_bridges
device_uuid => $on_device_uuid, device_uuid => $on_device_uuid,
}, },
}); });
($output, $return_code) = $anvil->Network->reset_connection({debug => 2, uuid => $on_device_uuid}); ($output, $return_code) = $anvil->Network->reset_connection({uuid => $on_device_uuid});
# Rescan. # Rescan.
$anvil->Job->update_progress({ $anvil->Job->update_progress({
@ -975,7 +1009,7 @@ sub reconfigure_bridges
line => __LINE__, line => __LINE__,
variables => { }, variables => { },
}); });
$anvil->Network->collect_data({debug => 2}); $anvil->Network->collect_data();
} }
} }
@ -1101,7 +1135,7 @@ sub reconfigure_bonds
}); });
# If there are ifcfg files for this bond, move it. # If there are ifcfg files for this bond, move it.
my $network_type = $anvil->System->check_network_type({debug => 2}); my $network_type = $anvil->System->check_network_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }});
if ($network_type eq "ifcfg") if ($network_type eq "ifcfg")
{ {
@ -1120,7 +1154,7 @@ sub reconfigure_bonds
job_uuid => $anvil->data->{job}{uuid}, job_uuid => $anvil->data->{job}{uuid},
variables => { file => $ifcfg_file }, variables => { file => $ifcfg_file },
}); });
$anvil->Storage->backup({debug => 2, file => $ifcfg_file}); $anvil->Storage->backup({file => $ifcfg_file});
unlink $ifcfg_file; unlink $ifcfg_file;
} }
} }
@ -1171,8 +1205,8 @@ sub reconfigure_bonds
device_uuid => $bond_uuid, device_uuid => $bond_uuid,
}, },
}); });
my ($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $bond_uuid, variable => "ipv4.method", value => "disabled"}); my ($output, $return_code) = $anvil->Network->modify_connection({uuid => $bond_uuid, variable => "ipv4.method", value => "disabled"});
($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $bond_uuid, variable => "ipv6.method", value => "disabled"}); ($output, $return_code) = $anvil->Network->modify_connection({uuid => $bond_uuid, variable => "ipv6.method", value => "disabled"});
my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$bond_name; my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$bond_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@ -1193,7 +1227,7 @@ sub reconfigure_bonds
file => $THIS_FILE, file => $THIS_FILE,
line => __LINE__, line => __LINE__,
}); });
$anvil->Network->collect_data({debug => 2}); $anvil->Network->collect_data();
} }
# Now add the interfaces, disabling their ipv4.method first. # Now add the interfaces, disabling their ipv4.method first.
@ -1290,8 +1324,8 @@ sub reconfigure_bonds
device_uuid => $nm_uuid, device_uuid => $nm_uuid,
}, },
}); });
my ($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $nm_uuid, variable => "ipv4.method", value => "disabled"}); my ($output, $return_code) = $anvil->Network->modify_connection({uuid => $nm_uuid, variable => "ipv4.method", value => "disabled"});
($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $nm_uuid, variable => "ipv6.method", value => "disabled"}); ($output, $return_code) = $anvil->Network->modify_connection({uuid => $nm_uuid, variable => "ipv6.method", value => "disabled"});
# Connecting the interface to the bond # Connecting the interface to the bond
$anvil->Job->update_progress({ $anvil->Job->update_progress({
@ -1349,7 +1383,7 @@ sub reconfigure_bonds
device_uuid => $nm_uuid, device_uuid => $nm_uuid,
}, },
}); });
($output, $return_code) = $anvil->Network->reset_connection({debug => 2, uuid => $nm_uuid}); ($output, $return_code) = $anvil->Network->reset_connection({uuid => $nm_uuid});
# Rescan. # Rescan.
$anvil->Job->update_progress({ $anvil->Job->update_progress({
@ -1362,7 +1396,7 @@ sub reconfigure_bonds
line => __LINE__, line => __LINE__,
variables => { }, variables => { },
}); });
$anvil->Network->collect_data({debug => 2}); $anvil->Network->collect_data();
} }
} }
} }
@ -1678,8 +1712,8 @@ sub reconfigure_ip_addresses
device_uuid => $old_uuid, device_uuid => $old_uuid,
}, },
}); });
my ($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $old_uuid, variable => "ipv4.method", value => "disabled"}); my ($output, $return_code) = $anvil->Network->modify_connection({uuid => $old_uuid, variable => "ipv4.method", value => "disabled"});
($output, $return_code) = $anvil->Network->modify_connection({debug => 2, uuid => $old_uuid, variable => "ipv6.method", value => "disabled"}); ($output, $return_code) = $anvil->Network->modify_connection({uuid => $old_uuid, variable => "ipv6.method", value => "disabled"});
$anvil->Job->update_progress({ $anvil->Job->update_progress({
progress => $anvil->Job->bump_progress({steps => 1}), progress => $anvil->Job->bump_progress({steps => 1}),
@ -1694,7 +1728,7 @@ sub reconfigure_ip_addresses
device_uuid => $old_uuid, device_uuid => $old_uuid,
}, },
}); });
($output, $return_code) = $anvil->Network->reset_connection({debug => 2, uuid => $old_uuid}); ($output, $return_code) = $anvil->Network->reset_connection({uuid => $old_uuid});
} }
# Now assign the IP. # Now assign the IP.
@ -1732,7 +1766,7 @@ sub reconfigure_ip_addresses
device_uuid => $on_device_uuid, device_uuid => $on_device_uuid,
}, },
}); });
($output, $return_code) = $anvil->Network->reset_connection({debug => 2, uuid => $on_device_uuid}); ($output, $return_code) = $anvil->Network->reset_connection({uuid => $on_device_uuid});
# Rescan. # Rescan.
$anvil->Job->update_progress({ $anvil->Job->update_progress({
@ -1745,7 +1779,7 @@ sub reconfigure_ip_addresses
line => __LINE__, line => __LINE__,
variables => { }, variables => { },
}); });
$anvil->Network->collect_data({debug => 2}); $anvil->Network->collect_data();
} }
} }
@ -1762,37 +1796,55 @@ sub reconfigure_interfaces
{ {
### TODO: Left off here... any devices that are down, set: ### TODO: Left off here... any devices that are down, set:
### nmcli connection modify <uuid> ipv4.method manual ipv4.addresses 169.0.0.x/8' (x == Wired connection x) ### nmcli connection modify <uuid> ipv4.method manual ipv4.addresses 169.0.0.x/8' (x == Wired connection x)
my $connection_id = $anvil->data->{nmcli}{uuid}{$uuid}{'connection.id'} // ""; my $connection_id = $anvil->data->{nmcli}{uuid}{$uuid}{'connection.id'} // "";
my $general_ip_iface = $anvil->data->{nmcli}{uuid}{$uuid}{'GENERAL.IP-IFACE'} // ""; my $general_ip_iface = $anvil->data->{nmcli}{uuid}{$uuid}{'GENERAL.IP-IFACE'} // "";
$general_ip_iface = "" if $general_ip_iface eq "--"; $general_ip_iface = "" if $general_ip_iface eq "--";
my $device_type = $anvil->data->{nmcli}{uuid}{$uuid}{'connection.type'} // ""; my $device_type = $anvil->data->{nmcli}{uuid}{$uuid}{'connection.type'} // "";
my $match_interface_name = $anvil->data->{nmcli}{uuid}{$uuid}{'match.interface-name'} // ""; my $match_interface_name = $anvil->data->{nmcli}{uuid}{$uuid}{'match.interface-name'} // "";
my $active = $anvil->data->{nmcli}{uuid}{$uuid}{active}; my $connection_autoconnect = $anvil->data->{nmcli}{uuid}{$uuid}{'connection.autoconnect'} // "";
my $active = $anvil->data->{nmcli}{uuid}{$uuid}{active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
's1:uuid' => $uuid, 's1:uuid' => $uuid,
's2:connection_id' => $connection_id, 's2:connection_id' => $connection_id,
's3:general_ip_iface' => $general_ip_iface, 's3:general_ip_iface' => $general_ip_iface,
's4:device_type' => $device_type, 's4:device_type' => $device_type,
's5:match_interface_name' => $match_interface_name, 's5:match_interface_name' => $match_interface_name,
's6:active' => $active, 's6:connection_autoconnect' => $connection_autoconnect,
's7:active' => $active,
}}); }});
# Try activating it. ### NOTE: I don't think this is needed, but here's the code in case we change our mind.
if ((not $general_ip_iface) && (not $active)) if (0)
{ {
# Try activating it.
# Rescan. if ((not $general_ip_iface) && (not $active))
$anvil->Job->update_progress({ {
progress => $anvil->Job->bump_progress({steps => 1}), $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0842", variables => {
message => "message_0394", id => $connection_id,
log_level => 1, uuid => $uuid,
'print' => 1, }});
job_uuid => $anvil->data->{job}{uuid},
file => $THIS_FILE, my $shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$uuid;
line => __LINE__, $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
variables => { }, my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
$anvil->Network->collect_data({debug => 2}); output => $output,
return_code => $return_code,
}});
# Rescan.
$anvil->Job->update_progress({
progress => $anvil->Job->bump_progress({steps => 1}),
message => "message_0394",
log_level => 1,
'print' => 1,
job_uuid => $anvil->data->{job}{uuid},
file => $THIS_FILE,
line => __LINE__,
variables => { },
});
$anvil->Network->collect_data();
}
} }
} }
@ -1885,6 +1937,16 @@ sub reconfigure_interfaces
$configure_link1 = 1; $configure_link1 = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configure_link1 => $configure_link1 }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configure_link1 => $configure_link1 }});
} }
# Make sure that the autoconnect is enabled.
if ($anvil->data->{nmcli}{uuid}{$link1_nm_uuid}{'connection.autoconnect'} ne "yes")
{
my ($output, $return_code) = $anvil->Network->modify_connection({uuid => $link1_nm_uuid, variable => "connection.autoconnect", value => "yes"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
}
} }
else else
{ {
@ -1935,6 +1997,16 @@ sub reconfigure_interfaces
$configure_link2 = 1; $configure_link2 = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configure_link1 => $configure_link1 }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { configure_link1 => $configure_link1 }});
} }
# Make sure that the autoconnect is enabled.
if ($anvil->data->{nmcli}{uuid}{$link2_nm_uuid}{'connection.autoconnect'} ne "yes")
{
my ($output, $return_code) = $anvil->Network->modify_connection({uuid => $link2_nm_uuid, variable => "connection.autoconnect", value => "yes"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
}
} }
else else
{ {
@ -2019,7 +2091,7 @@ sub rename_interface
}); });
# If there are ifcfg files for this device, move them. # If there are ifcfg files for this device, move them.
my $network_type = $anvil->System->check_network_type({debug => 2}); my $network_type = $anvil->System->check_network_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { network_type => $network_type }});
if ($network_type eq "ifcfg") if ($network_type eq "ifcfg")
{ {
@ -2045,7 +2117,7 @@ sub rename_interface
job_uuid => $anvil->data->{job}{uuid}, job_uuid => $anvil->data->{job}{uuid},
variables => { file => $file }, variables => { file => $file },
}); });
$anvil->Storage->backup({debug => 2, file => $file}); $anvil->Storage->backup({file => $file});
unlink $file; unlink $file;
} }
} }
@ -2056,7 +2128,7 @@ sub rename_interface
my $old_persistent_net = ""; my $old_persistent_net = "";
if (-e $anvil->data->{path}{configs}{'persistent-net'}) if (-e $anvil->data->{path}{configs}{'persistent-net'})
{ {
$old_persistent_net = $anvil->Storage->read_file({debug => 2, file => $anvil->data->{path}{configs}{'persistent-net'}}); $old_persistent_net = $anvil->Storage->read_file({file => $anvil->data->{path}{configs}{'persistent-net'}});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_persistent_net => $old_persistent_net }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_persistent_net => $old_persistent_net }});
} }
foreach my $line (split/\n/, $old_persistent_net) foreach my $line (split/\n/, $old_persistent_net)
@ -2280,7 +2352,7 @@ sub rename_interface
line => __LINE__, line => __LINE__,
variables => { }, variables => { },
}); });
$anvil->Network->collect_data({debug => 2}); $anvil->Network->collect_data();
# Set the reboot flag. # Set the reboot flag.
$anvil->data->{sys}{reboot_needed} = 1; $anvil->data->{sys}{reboot_needed} = 1;

@ -70,12 +70,15 @@ $anvil->System->_check_anvil_conf();
# If dnf is running, hold. # If dnf is running, hold.
$anvil->System->wait_on_dnf(); $anvil->System->wait_on_dnf();
# If we've got bonds, wait for them to be up. Then wait for NetworkManager to be up.
$anvil->Network->wait_on_nm_online({debug => 2});
$anvil->Network->wait_for_network({debug => 2});
# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks # Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server. # is to setup the database server.
$anvil->Database->connect({ $anvil->Database->connect({
check_if_configured => 1, check_if_configured => 1,
check_for_resync => 2, check_for_resync => 2,
debug => 2,
}); });
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "log_0132"});
@ -1751,10 +1754,10 @@ sub run_jobs
} }
# Convert the double-banged strings into a proper message. # Convert the double-banged strings into a proper message.
my $say_title = $job_title ? $anvil->Words->parse_banged_string({debug => 2, key_string => $job_title}) : ""; my $say_title = $job_title ? $anvil->Words->parse_banged_string({key_string => $job_title}) : "";
my $say_description = $job_description ? $anvil->Words->parse_banged_string({debug => 2, key_string => $job_description}) : ""; my $say_description = $job_description ? $anvil->Words->parse_banged_string({key_string => $job_description}) : "";
my $say_status = $job_status ? $anvil->Words->parse_banged_string({debug => 2, key_string => $job_status}) : ""; my $say_status = $job_status ? $anvil->Words->parse_banged_string({key_string => $job_status}) : "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_title => $job_title, job_title => $job_title,
say_description => $say_description, say_description => $say_description,
say_status => $say_status, say_status => $say_status,
@ -1898,7 +1901,7 @@ sub run_jobs
# Record that we've tried to start this job, so that we don't try to restart it for any reason for at least a minute. # Record that we've tried to start this job, so that we don't try to restart it for any reason for at least a minute.
$anvil->data->{jobs}{$job_uuid}{started} = time; $anvil->data->{jobs}{$job_uuid}{started} = time;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'jobs::$job_uuid::started' => $anvil->data->{jobs}{$job_uuid}{started} }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "jobs::${job_uuid}::started" => $anvil->data->{jobs}{$job_uuid}{started} }});
# Record that a job with this command has started # Record that a job with this command has started
$anvil->data->{jobs_started}{$short_command} = $job_uuid; $anvil->data->{jobs_started}{$short_command} = $job_uuid;

@ -36,10 +36,6 @@ $anvil->Get->switches({list => [
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
### TODO: Remove this post testing
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Database->connect(); $anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})

@ -55,9 +55,6 @@ if (not $anvil->data->{sys}{database}{connections})
# Get the job details # Get the job details
load_job($anvil); load_job($anvil);
# Make sure the hosts file has entries for all nets for both subnodes
wait_for_etc_hosts($anvil);
# Hold until both subnodes are marked as configured and not in maintenance mode. # Hold until both subnodes are marked as configured and not in maintenance mode.
wait_for_subnodes($anvil); wait_for_subnodes($anvil);
@ -67,6 +64,9 @@ update_passwords($anvil);
# Check if we need to change any IPs or our hostname. # Check if we need to change any IPs or our hostname.
check_local_network($anvil); check_local_network($anvil);
# Make sure the hosts file has entries for all nets for both subnodes
wait_for_etc_hosts($anvil);
# Wait until we can ping our peer on all networks. # Wait until we can ping our peer on all networks.
wait_for_access($anvil); wait_for_access($anvil);
@ -86,8 +86,8 @@ if (1)
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "anvil-safe-start.service" }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0094", variables => { daemon => "anvil-safe-start.service" }});
} }
update_progress($anvil, 100, "job_0128"); update_progress($anvil, 100, "job_0129");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0128"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0129"});
# Record that we're done configuring pacemaker. # Record that we're done configuring pacemaker.
$anvil->Database->insert_or_update_variables({ $anvil->Database->insert_or_update_variables({
@ -263,7 +263,7 @@ sub wait_for_etc_hosts
# Add the IP to be added to /etc/hosts. # Add the IP to be added to /etc/hosts.
$anvil->data->{to_add}{$host_name} = $ip_address; $anvil->data->{to_add}{$host_name} = $ip_address;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "to_add::${ip_address}" => $anvil->data->{to_add}{$host_name} }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "to_add::${host_name}" => $anvil->data->{to_add}{$host_name} }});
} }
} }
@ -279,108 +279,10 @@ sub wait_for_etc_hosts
{ {
# Not ready, wait a bit. # Not ready, wait a bit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0834"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0834"});
sleep 5;
# Add the values to /etc/hosts. # Try to update the /etc/hosts file
my $new_hosts_body = ""; $anvil->System->update_hosts({debug => 2});
foreach my $line (split/\n/, $hosts_file)
{
# See if this line needs to be modified.
if ($line =~ /^(\d.*?)\s+(.*)$/)
{
my $this_ip = $1;
my $hosts = $2;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
this_ip => $this_ip,
hosts => $hosts,
}});
if ($anvil->Validate->ip({ip => $this_ip}))
{
my $changes = 0;
my $new_hosts = "";
my $comment = ($hosts =~ /(#.*)$/)[0];
$comment = "" if not defined $comment;
$hosts =~ s/#.*?//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
comment => $comment,
hosts => $hosts,
}});
foreach my $host_name (split/\s+/, $hosts)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_name => $host_name }});
if ((exists $anvil->data->{to_add}{$host_name}) &&
($anvil->data->{to_add}{$host_name}) &&
($this_ip ne $anvil->data->{to_add}{$host_name}))
{
# Remove this host.
$changes = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changes => $changes }});
delete $anvil->data->{to_add}{$host_name};
next;
}
$new_hosts .= $host_name." ";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_hosts => $new_hosts }});
}
# Do we have any names to add to this IP?
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{to_add}})
{
if ($this_ip eq $anvil->data->{to_add}{$host_name})
{
# Add it.
$new_hosts .= $host_name." ";
$changes = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changes => $changes }});
delete $anvil->data->{to_add}{$host_name};
}
}
if ($changes)
{
$new_hosts_body .= $this_ip."\t".$new_hosts.$comment."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { changes => $changes }});
next;
}
}
}
# If we're alive here, just add the old line.
$new_hosts_body .= $line."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_hosts_body => $new_hosts_body }});
}
# Add any hosts still not processed.
foreach my $host_name (sort {$a cmp $b} keys %{$anvil->data->{to_add}})
{
$new_hosts_body .= $anvil->data->{to_add}{$host_name}."\t".$host_name."\n";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_hosts_body => $new_hosts_body }});
delete $anvil->data->{to_add}{$host_name};
}
# If there's a difference, write the new hosts file.
my $difference = diff \$new_hosts_body, \$hosts_file, { STYLE => 'Unified' };
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
if ($difference)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 3, key => "job_0112"});
update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0112");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0835"});
my $failed = $anvil->Storage->write_file({
debug => 2,
overwrite => 1,
file => $anvil->data->{path}{configs}{hosts},
body => $new_hosts_body,
user => "root",
group => "root",
mode => "0644",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
}
sleep 1; sleep 1;
} }
} }
@ -1413,19 +1315,31 @@ sub configure_pacemaker
}}); }});
if ($return_code) if ($return_code)
{ {
# Something went wrong. # See if this was because the fence method already existed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => { if ($output =~ /already exists/)
shell_call => $shell_call, {
output => $output, # It already existed, we're fine.
return_code => $return_code, update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0481,!!device!".$stonith_name."!!");
}}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0120", variables => { device => $stonith_name }});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!"); }
sleep 2; else
$anvil->nice_exit({exit_code => 6}); {
# Something went wrong.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0138", variables => {
shell_call => $shell_call,
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 0, "error_0138,!!shell_call!".$shell_call."!!,!!output!".$output."!!,!!return_code!".$return_code."!!");
sleep 2;
$anvil->nice_exit({exit_code => 6});
}
}
else
{
$something_changed->{$node_name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
} }
$something_changed->{$node_name} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "something_changed->{$node_name}" => $something_changed->{$node_name} }});
} }
} }

@ -0,0 +1,390 @@
#!/usr/bin/perl
#
# This daemon monitors and logs preformance data. This is meant to help debug issues related to (potential)
# performance issues.
#
# NOTE: This is designed to be minimal overhead, so there is no attempt to connect to the database. As such,
# be mindful of what this daemon is used for.
#
use strict;
use warnings;
use Data::Dumper;
use Anvil::Tools;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
# Read switches
$anvil->Get->switches({list => [
"enable",
"disable",
"log-only",
"monitor",
"now",
"start",
"status",
"stop",
], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
# Common daemons
$anvil->data->{daemons}{core}{'anvil-daemon'} = "anvil-daemon.service";
$anvil->data->{daemons}{core}{'anvil-monitor-daemons'} = "anvil-monitor-daemons.service";
$anvil->data->{daemons}{core}{'anvil-monitor-network'} = "anvil-monitor-network.service";
$anvil->data->{daemons}{core}{'anvil-monitor-performance'} = "anvil-monitor-performance.service";
$anvil->data->{daemons}{core}{'scancore'} = "scancore.service";
# Striker dashboards.
$anvil->data->{daemons}{striker}{'striker-ui-api'} = "striker-ui-api.service";
# Nodes and DR hosts
$anvil->data->{daemons}{node}{'anvil-safe-start'} = "anvil-safe-start.service";
$anvil->data->{daemons}{dr}{'anvil-safe-start'} = "anvil-safe-start.service";
my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { host_type => $host_type }});
my $daemon_list = [];
foreach my $daemon (sort {$a cmp $b} keys %{$anvil->data->{daemons}{core}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { daemon => $daemon }});
push @{$daemon_list}, $daemon;
}
foreach my $daemon (sort {$a cmp $b} keys %{$anvil->data->{daemons}{$host_type}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { daemon => $daemon }});
push @{$daemon_list}, $daemon;
}
if (($anvil->data->{switches}{monitor}) or ($anvil->data->{switches}{'log-only'}))
{
# Run as a daemon
monitor_daemons($anvil);
}
elsif ($anvil->data->{switches}{status})
{
report_status($anvil, "stdout");
}
else
{
# We're staritng, stopping, enabling or disabling.
if (($anvil->data->{switches}{enable}) or
($anvil->data->{switches}{disable}) or
($anvil->data->{switches}{start}) or
($anvil->data->{switches}{stop}))
{
check_daemon($anvil);
process_daemons($anvil);
}
else
{
# Bad call
print "Please see 'man ".$THIS_FILE." for usage.\n";
$anvil->nice_exit({exit_code => 1});
}
}
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
sub report_status
{
my ($anvil, $target) = @_;
# Return Code meanings:
# - Enabled = 0, Disabled = 1
# - Started = 0, Stopped = 3
# - Failed = 0, OK = 1,
check_daemon($anvil);
my $print = $target eq "stdout" ? 1 : 0;
# Only print the header if we're printing to STDOUT
if ($print)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, secure => 0, key => "message_0419"});
}
my $longest_daemon = $anvil->data->{longest_daemon};
my $string = "";
foreach my $daemon (sort {$a cmp $b} keys %{$anvil->data->{daemon}})
{
my $say_enabled = "unknown";
my $say_started = "unknown";
my $say_failed = "unknown";
# Enabled/Disabled
if ($anvil->data->{daemon}{$daemon}{enabled}{return_code} == 0)
{
$say_enabled = "Enabled";
}
elsif ($anvil->data->{daemon}{$daemon}{enabled}{return_code} == 1)
{
$say_enabled = "Disabled";
}
# Started / Stopped
if ($anvil->data->{daemon}{$daemon}{active}{return_code} == 0)
{
$say_started = "Started";
}
elsif ($anvil->data->{daemon}{$daemon}{active}{return_code} == 3)
{
$say_started = "Stopped";
}
# Failed / OK
if ($anvil->data->{daemon}{$daemon}{failed}{return_code} == 0)
{
$say_failed = "Failed!";
}
elsif ($anvil->data->{daemon}{$daemon}{failed}{return_code} == 1)
{
$say_failed = "OK";
}
#my $say_daemon = sprintf("%-${longest_daemon}s", $daemon.";");
my $say_daemon = $daemon."; ";
my $dots = $longest_daemon - (length($daemon));
for (my $i = 0; $i < $dots; $i++)
{
$say_daemon .= ".";
}
$string .= $daemon." [".$say_enabled."/".$say_started."/".$say_failed."], ";
### NOTE: Set this to '3' because it was filling the logs when '--debug' is used.
# Report.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 3, secure => 0, key => "message_0420", variables => {
daemon => $say_daemon,
enabled => $say_enabled,
enable_string => $anvil->data->{daemon}{$daemon}{enabled}{string},
started => $say_started,
start_string => $anvil->data->{daemon}{$daemon}{active}{string},
failed => $say_failed,
fail_string => $anvil->data->{daemon}{$daemon}{failed}{string},
}});
}
$string =~ s/, $//gs;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, secure => 0, key => "message_0421", variables => { string => $string }});
return(0);
}
sub check_daemon
{
my ($anvil) = @_;
$anvil->data->{longest_daemon} = 0;
foreach my $daemon (sort {$a cmp $b} @{$daemon_list})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { daemon => $daemon }});
if (length($daemon) > $anvil->data->{longest_daemon})
{
$anvil->data->{longest_daemon} = length($daemon);
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { longest_daemon => $anvil->data->{longest_daemon} }});
}
# What's the status of this daemon?
foreach my $test ("active", "enabled", "failed")
{
my $shell_call = $anvil->data->{path}{exe}{systemctl}." is-".$test." ".$daemon;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
output => $output,
return_code => $return_code,
}});
# There should only be one line of output.
$anvil->data->{daemon}{$daemon}{$test}{string} = $output;
$anvil->data->{daemon}{$daemon}{$test}{return_code} = $return_code;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"daemon::${daemon}::${test}::string" => $anvil->data->{daemon}{$daemon}{$test}{string},
"daemon::${daemon}::${test}::return_code" => $anvil->data->{daemon}{$daemon}{$test}{return_code},
}});
}
}
foreach my $daemon (sort {$a cmp $b} keys %{$anvil->data->{daemon}})
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { daemon => $daemon }});
foreach my $test (sort {$a cmp $b} keys %{$anvil->data->{daemon}{$daemon}})
{
my $string = $anvil->data->{daemon}{$daemon}{$test}{string};
my $return_code = $anvil->data->{daemon}{$daemon}{$test}{return_code};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:test" => $test,
"s2:string" => $string,
"s3:return_code" => $return_code,
}});
}
}
return(0);
}
sub process_daemons
{
my ($anvil) = @_;
# If enabling now, set the start switch, and inverse for disable
if (($anvil->data->{switches}{enable}) && ($anvil->data->{switches}{now}) && (not $anvil->data->{switches}{start}))
{
print "Will start any stopped daemons ('--enable --now' used) after enabling daemons.\n";
$anvil->data->{switches}{start} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { "switches::start" => $anvil->data->{switches}{start} }});
}
elsif (($anvil->data->{switches}{disable}) && ($anvil->data->{switches}{now}) && (not $anvil->data->{switches}{stop}))
{
print "Will stop any running daemons ('--disable --now' used) after disabling daemons.\n";
$anvil->data->{switches}{stop} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { "switches::stop" => $anvil->data->{switches}{stop} }});
}
# Return Code meanings:
# - Enabled = 0, Disabled = 1
# - Started = 0, Stopped = 3
# - Failed = 0, OK = 1,
foreach my $daemon (sort {$a cmp $b} keys %{$anvil->data->{daemon}})
{
# Enable or disable?
if (($anvil->data->{switches}{enable}) && ($anvil->data->{daemon}{$daemon}{enabled}{return_code} == 1))
{
my $shell_call = $anvil->data->{path}{exe}{systemctl}." enable ".$daemon;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { shell_call => $shell_call }});
print "Enabling: [".$daemon."] now...\n";
my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
output => $output,
return_code => $return_code,
}});
}
elsif (($anvil->data->{switches}{disable}) && ($anvil->data->{daemon}{$daemon}{enabled}{return_code} == 0))
{
my $shell_call = $anvil->data->{path}{exe}{systemctl}." disable ".$daemon;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { shell_call => $shell_call }});
print "Disabling: [".$daemon."] now...\n";
my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
output => $output,
return_code => $return_code,
}});
}
# Start or stop?
if (($anvil->data->{switches}{start}) && ($anvil->data->{daemon}{$daemon}{active}{return_code} == 3))
{
my $shell_call = $anvil->data->{path}{exe}{systemctl}." start ".$daemon;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { shell_call => $shell_call }});
print "Starting: [".$daemon."] now...\n";
my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
output => $output,
return_code => $return_code,
}});
}
elsif (($anvil->data->{switches}{stop}) && ($anvil->data->{daemon}{$daemon}{active}{return_code} == 0))
{
my $shell_call = $anvil->data->{path}{exe}{systemctl}." stop ".$daemon;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { shell_call => $shell_call }});
print "Stopping: [".$daemon."] now...\n";
my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
output => $output,
return_code => $return_code,
}});
}
}
print "Done.\n\n";
sleep 1;
report_status($anvil, "stdout");
return(0);
}
sub monitor_daemons
{
my ($anvil) = @_;
# Calculate my sum so that we can exit if it changes later.
$anvil->Storage->record_md5sums;
my $next_md5sum_check = time + 30;
while(1)
{
if (exists $anvil->data->{daemon})
{
delete $anvil->data->{daemon};
}
check_daemon($anvil);
if ($anvil->data->{switches}{monitor})
{
foreach my $daemon (sort {$a cmp $b} keys %{$anvil->data->{daemon}})
{
# Return code of '1' is OK, '0' is failed.
my $string = $anvil->data->{daemon}{$daemon}{failed}{string};
my $return_code = $anvil->data->{daemon}{$daemon}{failed}{return_code};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
"s1:daemon" => $daemon,
"s2:string" => $string,
"s3:return_code" => $return_code,
}});
if (not $return_code)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, secure => 0, key => "warning_0171", variables => { daemon => $daemon }});
my $shell_call = $anvil->data->{path}{exe}{systemctl}." restart ".$daemon;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => {
output => $output,
return_code => $return_code,
}});
}
}
}
else
{
report_status($anvil, "log");
}
if (time > $next_md5sum_check)
{
$next_md5sum_check = time + 30;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { next_md5sum_check => $next_md5sum_check }});
if ($anvil->Storage->check_md5sums)
{
# NOTE: We exit with '0' to prevent systemctl from showing a scary red message.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "message_0014"});
$anvil->nice_exit({exit_code => 0});
}
}
sleep 60;
}
return(0);
}

@ -0,0 +1,159 @@
#!/usr/bin/perl
#
# This daemon monitors and logs preformance data. This is meant to help debug issues related to (potential)
# performance issues.
#
# NOTE: This is designed to be minimal overhead, so there is no attempt to connect to the database. As such,
# be mindful of what this daemon is used for.
#
use strict;
use warnings;
use Data::Dumper;
use Anvil::Tools;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete.
$| = 1;
my $anvil = Anvil::Tools->new();
# Read switches
$anvil->Get->switches({list => [
"detailed",
"interval",
"print",
"run-once",
], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
# Calculate my sum so that we can exit if it changes later.
$anvil->Storage->record_md5sums;
my $next_md5sum_check = time + 30;
our $interval = $anvil->data->{switches}{interval} =~ /^\d+$/ ? $anvil->data->{switches}{interval} : 5;
our $print = $anvil->data->{switches}{'print'} ? 1 : 0;
our $detailed = $anvil->data->{switches}{detailed} ? 1 : 0;
# Now go into the main loop
while(1)
{
my $scan_time = time;
record_data($anvil);
if ($anvil->data->{switches}{'run-once'})
{
# We're done.
$anvil->nice_exit({exit_code => 0});
}
if (time > $next_md5sum_check)
{
$next_md5sum_check = time + 30;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { next_md5sum_check => $next_md5sum_check }});
if ($anvil->Storage->check_md5sums)
{
# NOTE: We exit with '0' to prevent systemctl from showing a scary red message.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "message_0014"});
$anvil->nice_exit({exit_code => 0});
}
}
sleep $interval;
}
sub record_data
{
my ($anvil) = @_;
$anvil->Get->load_average({debug => 3});
if ($detailed)
{
# Show the CPU load percents.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, key => "log_0840", variables => {
iowait => $anvil->data->{loads}{load_percent}{iowait},
user => $anvil->data->{loads}{load_percent}{user},
steal => $anvil->data->{loads}{load_percent}{steal},
idle => $anvil->data->{loads}{load_percent}{idle},
nice => $anvil->data->{loads}{load_percent}{nice},
'system' => $anvil->data->{loads}{load_percent}{'system'},
}});
# Show the CPU load levels
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, key => "log_0835", variables => {
one_minute => $anvil->data->{loads}{load_average}{one_minute},
five_minutes => $anvil->data->{loads}{load_average}{five_minute},
ten_minutes => $anvil->data->{loads}{load_average}{ten_minute},
}});
# Show the processes
my $key = $detailed ? "log_0841" : "log_0836";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, key => $key, variables => {
total => $anvil->Convert->add_commas({number => $anvil->data->{loads}{processes}{total}}),
running => $anvil->Convert->add_commas({number => $anvil->data->{loads}{processes}{running}}),
blocked => $anvil->Convert->add_commas({number => $anvil->data->{loads}{processes}{blocked}}),
interrupts => $anvil->Convert->add_commas({number => $anvil->data->{loads}{interrupts}{total}}),
}});
# CPU average load times
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, key => "log_0837", variables => {
io_wait => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{average}{io_wait} / 100)}),
user_mode => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{average}{user_mode} / 100)}),
user_mode_nice => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{average}{user_mode_nice} / 100)}),
system_mode => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{average}{system_mode} / 100)}),
idle_tasks => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{average}{idle_tasks} / 100)}),
hard_irq => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{average}{hard_irq} / 100)}),
soft_irq => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{average}{soft_irq} / 100)}),
}});
# Show per-cores
foreach my $core (sort {$a <=> $b} keys %{$anvil->data->{loads}{cpu}{core}})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, key => "log_0838", variables => {
core => $core,
user_mode => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{core}{$core}{user_mode} / 100)}),
user_mode_nice => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{core}{$core}{user_mode_nice} / 100)}),
system_mode => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{core}{$core}{system_mode} / 100)}),
idle_tasks => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{core}{$core}{idle_tasks} / 100)}),
hard_irq => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{core}{$core}{hard_irq} / 100)}),
soft_irq => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{cpu}{core}{$core}{soft_irq} / 100)}),
}});
}
# This is the number of IO operations in progress. When IOs in progress is non-zero, the weighted time (in 1/100ths of a second), doing those IOs.
foreach my $device_name (sort {$a cmp $b} keys %{$anvil->data->{loads}{storage}})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, key => "log_0839", variables => {
device_name => $device_name,
in_progress => $anvil->data->{loads}{storage}{$device_name}{ios_currently_in_progress},
weighted_time_spent => $anvil->Convert->add_commas({number => ($anvil->data->{loads}{storage}{$device_name}{weighted_time_spent_doing_ios} / 100)}),
}});
}
}
else
{
# This is much more condensed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => $print, level => 1, key => "log_0846", variables => {
one_minute => $anvil->data->{loads}{load_average}{one_minute},
five_minutes => $anvil->data->{loads}{load_average}{five_minute},
ten_minutes => $anvil->data->{loads}{load_average}{ten_minute},
iowait => $anvil->data->{loads}{load_percent}{iowait},
running => $anvil->Convert->add_commas({number => $anvil->data->{loads}{processes}{running}}),
blocked => $anvil->Convert->add_commas({number => $anvil->data->{loads}{processes}{blocked}}),
}});
}
return(0);
}

@ -99,6 +99,54 @@ $anvil->nice_exit({exit_code => 0});
# Functions # # Functions #
############################################################################################################# #############################################################################################################
# This checks to see if any storage groups exist yet. If they don't, an attempt to assemble the storage
# groups will be made
sub check_Storage_groups
{
my ($anvil) = @_;
# Make sure we have an anvil_uuid
if ($anvil->data->{new_server}{anvil_uuid})
{
# Nope, nothing to do.
return(1);
}
my $anvil_uuid = $anvil->data->{new_server}{anvil_uuid};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_uuid => $anvil_uuid }});
# Load Storage Group data
$anvil->Database->get_storage_group_data({debug => 2});
if (exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid})
{
my $count = 0;
if ((exists $anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}) &&
(ref($anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}) eq "HASH"))
{
$count = keys %{$anvil->data->{storage_groups}{anvil_uuid}{$anvil_uuid}{storage_group_name}};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
}
if ($count)
{
# There's a storage group already, nothing to do.
return(0);
}
# Still alive? Try to assemble a storage group.
$anvil->Cluster->assemble_storage_groups({
debug => 2,
anvil_uuid => $anvil_uuid,
});
# Reload in case we created one.
$anvil->Database->get_storage_group_data({debug => 2});
}
return(0);
}
# This actually provisions a VM. # This actually provisions a VM.
sub run_jobs sub run_jobs
{ {

@ -44,11 +44,6 @@ $anvil->Get->switches({list => [], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
### TODO: Remove before PR!
# Force up the logging.
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
# Make sure we're running as 'root' # Make sure we're running as 'root'
# $< == real UID, $> == effective UID # $< == real UID, $> == effective UID
if (($< != 0) && ($> != 0)) if (($< != 0) && ($> != 0))

@ -35,10 +35,6 @@ my $anvil = Anvil::Tools->new();
$anvil->data->{switches}{'job-uuid'} = ""; $anvil->data->{switches}{'job-uuid'} = "";
$anvil->Get->switches; $anvil->Get->switches;
### TODO: Remove this
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
# Connect to the database(s). # Connect to the database(s).

@ -70,6 +70,10 @@ $anvil->Storage->read_config();
# If dnf is running, hold. # If dnf is running, hold.
$anvil->System->wait_on_dnf(); $anvil->System->wait_on_dnf();
# If we've got bonds, wait for them to be up. Then wait for NetworkManager to be up.
$anvil->Network->wait_on_nm_online({debug => 2});
$anvil->Network->wait_for_network({debug => 2});
# Read switches # Read switches
$anvil->Get->switches({list => ["purge", "run-once"], man => $THIS_FILE}); $anvil->Get->switches({list => ["purge", "run-once"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
@ -140,6 +144,9 @@ while(1)
# Do we have at least one database? # Do we have at least one database?
my $agent_runtime = 0; my $agent_runtime = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"sys::database::connections" => $anvil->data->{sys}{database}{connections},
}});
if ($anvil->data->{sys}{database}{connections}) if ($anvil->data->{sys}{database}{connections})
{ {
# Run the normal tasks # Run the normal tasks

@ -21,9 +21,33 @@ if (($running_directory =~ /^\./) && ($ENV{PWD}))
$running_directory =~ s/^\./$ENV{PWD}/; $running_directory =~ s/^\./$ENV{PWD}/;
} }
my $anvil = Anvil::Tools->new(); my $anvil = Anvil::Tools->new();
# Before we read switches, set the debug file.
if (not -e $anvil->data->{path}{configs}{'anvil.debug'})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "log_0843", variables => {
file => $anvil->data->{path}{configs}{'anvil.debug'},
host => $anvil->Get->short_host_name(),
}});
my $problem = $anvil->Storage->write_file({
file => $anvil->data->{path}{configs}{'anvil.debug'},
body => "",
overwrite => 1,
user => "root",
group => "root",
mode => "0666",
});
}
# Read switches
$anvil->Get->switches({list => [
"config",
"job-uuid",
], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Database->connect; $anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
@ -32,9 +56,6 @@ if (not $anvil->data->{sys}{database}{connections})
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, priority => "err", key => "error_0003"});
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }
$anvil->data->{switches}{config} = "";
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->Get->switches;
# Read in the config file # Read in the config file
if ((not $anvil->data->{switches}{config}) or (not -f $anvil->data->{switches}{config})) if ((not $anvil->data->{switches}{config}) or (not -f $anvil->data->{switches}{config}))
@ -324,7 +345,7 @@ sub configure_machine_networks
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2, debug => 2,
job_host_uuid => $machine_host_uuid, job_host_uuid => $machine_host_uuid,
job_command => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches, job_command => $anvil->data->{path}{exe}{'anvil-configure-host'}." --debug ".$anvil->Log->switches,
job_data => "form::config_step2", job_data => "form::config_step2",
job_name => "configure::network", job_name => "configure::network",
job_title => "job_0001", job_title => "job_0001",
@ -1683,7 +1704,7 @@ sub striker_stage1
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2, debug => 2,
job_host_uuid => $anvil->data->{cgi}{host_uuid}{value}, job_host_uuid => $anvil->data->{cgi}{host_uuid}{value},
job_command => $anvil->data->{path}{exe}{'anvil-configure-host'}.$anvil->Log->switches, job_command => $anvil->data->{path}{exe}{'anvil-configure-host'}." --debug ".$anvil->Log->switches,
job_data => "form::config_step2", job_data => "form::config_step2",
job_name => "configure::network", job_name => "configure::network",
job_title => "job_0001", job_title => "job_0001",
@ -1698,7 +1719,7 @@ sub striker_stage1
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({ my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
debug => 2, debug => 2,
job_host_uuid => $anvil->data->{cgi}{host_uuid}{value}, job_host_uuid => $anvil->data->{cgi}{host_uuid}{value},
job_command => $anvil->data->{path}{exe}{'striker-auto-initialize-all'}." --config ".$anvil->data->{switches}{config}.$anvil->Log->switches, job_command => $anvil->data->{path}{exe}{'striker-auto-initialize-all'}." --debug --config ".$anvil->data->{switches}{config}.$anvil->Log->switches,
job_data => "", job_data => "",
job_name => "configure::auto_initialize", job_name => "configure::auto_initialize",
job_title => "job_0225", job_title => "job_0225",

@ -40,10 +40,6 @@ $anvil->Get->switches({list => [
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
### TODO: Disable this before merging into PR
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
# Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks # Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks
# is to setup the database server. # is to setup the database server.
$anvil->Database->connect(); $anvil->Database->connect();
@ -361,7 +357,7 @@ sub collect_remote_data
source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/tmp/journalctl-previous-boot.log", source => "root\@".$anvil->data->{peer}{$short_host_name}{access}{ip}.":/tmp/journalctl-previous-boot.log",
destination => $target_directory."/", destination => $target_directory."/",
}); });
my $test_file = $target_directory."/tmp/journalctl-previous-boot.log"; my $test_file = $target_directory."/journalctl-previous-boot.log";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
if (-e $test_file) if (-e $test_file)
{ {
@ -370,7 +366,7 @@ sub collect_remote_data
else else
{ {
print "Failed!\n"; print "Failed!\n";
print "- For some reason, this file was not collected.\n"; print " - [ Warning ] - For some reason, this file was not collected.\n";
$anvil->Storage->write_file({ $anvil->Storage->write_file({
file => $test_file, file => $test_file,
body => $failed_body, body => $failed_body,
@ -481,7 +477,7 @@ sub collect_remote_data
overwrite => 0, overwrite => 0,
backup => 0, backup => 0,
}); });
my $shell_call = $anvil->data->{path}{exe}{pg_dump}." -h ".$anvil->data->{peer}{$short_host_name}{access}{ip}." -U admin anvil 2>/dev/null | ".$anvil->data->{path}{exe}{bzip2}." --stdout > ".$target_directory."/anvil.out.bz2"; my $shell_call = $anvil->data->{path}{exe}{pg_dump}." -h ".$anvil->data->{peer}{$short_host_name}{access}{ip}." -U admin anvil > ".$target_directory."/anvil.out.bz2";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});

@ -3,7 +3,9 @@ MAINTAINERCLEANFILES = Makefile.in
servicedir = $(SYSTEMD_UNIT_DIR) servicedir = $(SYSTEMD_UNIT_DIR)
dist_service_DATA = \ dist_service_DATA = \
anvil-daemon.service \ anvil-daemon.service \
anvil-monitor-daemons.service \
anvil-monitor-network.service \ anvil-monitor-network.service \
anvil-monitor-performance.service \
anvil-safe-start.service \ anvil-safe-start.service \
scancore.service \ scancore.service \
striker-ui-api.service striker-ui-api.service

@ -1,6 +1,6 @@
[Unit] [Unit]
Description=Anvil! Intelligent Availability Platform - main service daemon Description=Anvil! Intelligent Availability Platform - main service daemon
Wants=network.target After=network.target
[Service] [Service]
Type=simple Type=simple

@ -0,0 +1,13 @@
[Unit]
Description=Anvil! Intelligent Availability Platform - Daemon Monitor
Wants=network.target
[Service]
Type=simple
ExecStart=/usr/sbin/anvil-manage-daemons --log-only
ExecStop=/bin/kill -WINCH ${MAINPID}
Restart=always
RestartSec=60
[Install]
WantedBy=multi-user.target

@ -0,0 +1,12 @@
[Unit]
Description=Anvil! Intelligent Availability Platform - Performance Monitor Daemon
Wants=network.target
[Service]
Type=simple
ExecStart=/usr/sbin/anvil-monitor-performance
ExecStop=/bin/kill -WINCH ${MAINPID}
Restart=always
[Install]
WantedBy=multi-user.target

@ -1,6 +1,6 @@
[Unit] [Unit]
Description=Anvil! Intelligent Availability Platform - ScanCore Decision Engine Description=Anvil! Intelligent Availability Platform - ScanCore Decision Engine
Wants=network.target After=network.target
[Service] [Service]
Type=simple Type=simple

Loading…
Cancel
Save