From 53d654fd9dc4749a37a81050d1931127171b3f83 Mon Sep 17 00:00:00 2001 From: Digimer Date: Fri, 19 Mar 2021 18:19:29 -0400 Subject: [PATCH] * Got scan-filesystem to the point where it now collects the data it needs. No processing done yet though. Also reworked the SQL schema for the agent to record more data. * Created Storage->parse_df that, shock!, parses 'df' output. Finished the long-ago started ->parse_lsblk as well. Signed-off-by: Digimer --- Anvil/Tools/Storage.pm | 268 ++++++++++++++++-- Anvil/Tools/System.pm | 148 ++++++++++ .../scan-filesystems/scan-filesystems | 219 +++++++++----- .../scan-filesystems/scan-filesystems.sql | 50 ++-- scancore-agents/scan-hardware/scan-hardware | 2 +- share/words.xml | 2 + 6 files changed, 564 insertions(+), 125 deletions(-) diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 555123b3..5430623f 100644 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -6,9 +6,10 @@ package Anvil::Tools::Storage; use strict; use warnings; use Data::Dumper; +use File::MimeInfo; +use JSON; use Scalar::Util qw(weaken isweak); use Text::Diff; -use File::MimeInfo; use utf8; our $VERSION = "3.0.0"; @@ -25,6 +26,7 @@ my $THIS_FILE = "Storage.pm"; # get_storage_group_details # make_directory # move_file +# parse_df # parse_lsblk # read_config # read_file @@ -2031,32 +2033,98 @@ fi"; } -=head2 parse_lsblk - -This calls C<< lsblk >> (in json format) and parses the output. Data is stored as: - - * lsblk::::... - -Parameters; +=head2 parse_df -=head3 password (optional) +This calls C<< df >> and parses the output. Data is stored as: -If C<< target >> is set, this is the password used to log into the remote system as the C<< remote_user >>. If it is not set, an attempt to connect without a password will be made.. + * storage::df::::... -=head3 port (optional, default 22) +This method takes no parameters. -If C<< target >> is set, this is the TCP port number used to connect to the remote machine. - -=head3 remote_user (optional) +=cut +sub parse_df +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->parse_df()" }}); + + my $shell_call = $anvil->data->{path}{exe}{df}." --exclude-type=tmpfs --exclude-type=devtmpfs --no-sync --block-size=1 --output=source,fstype,size,used,avail,target"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); + + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + if ($return_code) + { + # Failed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0079", variables => { + return_code => $return_code, + output => $output, + }}); + return(1); + } + + foreach my $line (split/\n/, $output) + { + if ($line =~ /^\/dev\/(.*?)\s+(.*?)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\/.*)$/) + { + my $kernel_device_name = $1; + my $filesystem_type = $2; + my $size = $3; + my $used = $4; + my $free = $5; + my $mount_point = $6; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + 's1:kernel_device_name' => $kernel_device_name, + 's2:mount_point' => $mount_point, + 's3:filesystem_type' => $filesystem_type, + 's4:size' => $anvil->Convert->add_commas({number => $size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size}).")", + 's5:used' => $anvil->Convert->add_commas({number => $used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $used}).")", + 's6:free' => $anvil->Convert->add_commas({number => $free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $free}).")", + }}); + + # If the line starts with 'mapper', we need to figure out what dm-X device it is. + if ($kernel_device_name =~ /^mapper\//) + { + # Use lstat + my $device_path = "/dev/".$kernel_device_name; + my $device_mapper = readlink($device_path); + if ($device_mapper =~ /^\.\.\/(.*)$/) + { + $kernel_device_name = $1; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + device_path => $device_path, + kernel_device_name => $kernel_device_name, + }}); + } + + $anvil->{storage}{df}{$kernel_device_name}{filesystem_type} = $filesystem_type; + $anvil->{storage}{df}{$kernel_device_name}{mount_point} = $mount_point; + $anvil->{storage}{df}{$kernel_device_name}{size} = $size; + $anvil->{storage}{df}{$kernel_device_name}{used} = $used; + $anvil->{storage}{df}{$kernel_device_name}{free} = $free; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "storage::df::${kernel_device_name}::filesystem_type" => $anvil->{storage}{df}{$kernel_device_name}{filesystem_type}, + "storage::df::${kernel_device_name}::mount_point" => $anvil->{storage}{df}{$kernel_device_name}{mount_point}, + "storage::df::${kernel_device_name}::size" => $anvil->Convert->add_commas({number => $anvil->{storage}{df}{$kernel_device_name}{size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->{storage}{df}{$kernel_device_name}{size}}).")", + "storage::df::${kernel_device_name}::used" => $anvil->Convert->add_commas({number => $anvil->{storage}{df}{$kernel_device_name}{used}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->{storage}{df}{$kernel_device_name}{used}}).")", + "storage::df::${kernel_device_name}::free" => $anvil->Convert->add_commas({number => $anvil->{storage}{df}{$kernel_device_name}{free}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->{storage}{df}{$kernel_device_name}{free}}).")", + }}); + } + } + + return(0); +} -If C<< target >> is set, this is the user account that will be used when connecting to the remote system. -=head3 target (optional) +=head2 parse_lsblk -If set, C<< lsblk >> read from the target machine. This must be either an IP address or a resolvable host name. +This calls C<< lsblk >> (in json format) and parses the output. Data is stored as: -B<< Note >>: If not set, the short host name of this system is used in C<< lsblk::::: >>. + * storage::lsblk::::... +This method takes no parameters. =cut sub parse_lsblk { @@ -2066,21 +2134,159 @@ sub parse_lsblk my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Storage->parse_lsblk()" }}); - # Setup default values - my $password = defined $parameter->{password} ? $parameter->{password} : ""; - my $port = defined $parameter->{port} ? $parameter->{port} : 22; - my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : "root"; - my $target = defined $parameter->{target} ? $parameter->{target} : ""; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - password => $anvil->Log->is_secure($password), - port => $port, - remote_user => $remote_user, - target => $target, - }}); + my $shell_call = $anvil->data->{path}{exe}{lsblk}." --output KNAME,FSTYPE,MOUNTPOINT,UUID,PARTLABEL,PARTUUID,RO,RM,HOTPLUG,MODEL,SERIAL,SIZE,STATE,ALIGNMENT,PHY-SEC,LOG-SEC,ROTA,SCHED,TYPE,TRAN,VENDOR --bytes --json"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); - my $shell_call = $anvil->data->{path}{exe}{lsblk}." --all --bytes --json"; - + my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); + if ($return_code) + { + # Failed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0079", variables => { + return_code => $return_code, + output => $output, + }}); + return(1); + } + + my $json = JSON->new->allow_nonref; + my $data = $json->decode($output); + + foreach my $hash_ref (@{$data->{blockdevices}}) + { + my $kernel_device_name = $hash_ref->{kname}; + #next if $kernel_device_name =~ /^dm-/; + #next if $kernel_device_name =~ /^mmcblk/; # Add support for this later when 'System->parse_lshw' is done + $anvil->{storage}{lsblk}{$kernel_device_name}{alignment_offset} = defined $hash_ref->{alignment} ? $hash_ref->{alignment} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{device_type} = defined $hash_ref->{type} ? $hash_ref->{type} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_type} = defined $hash_ref->{fstype} ? $hash_ref->{fstype} : ""; + # This is the LVM formatted UUID, when it's an 'LVM2_member', so it should be easy to cross + # reference with: scan_lvm_lvs -> scan_lvm_lv_internal_uuid to map the LVs to a PV + $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_uuid} = defined $hash_ref->{uuid} ? $hash_ref->{uuid} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{hot_plug} = defined $hash_ref->{hotplug} ? $hash_ref->{hotplug} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{logical_sector_size} = defined $hash_ref->{'log-sec'} ? $hash_ref->{'log-sec'} : 0; + $anvil->{storage}{lsblk}{$kernel_device_name}{model} = defined $hash_ref->{model} ? $hash_ref->{model} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{model} = $anvil->Words->clean_spaces({string => $anvil->{storage}{lsblk}{$kernel_device_name}{model}}); + $anvil->{storage}{lsblk}{$kernel_device_name}{mount_point} = defined $hash_ref->{mountpoint} ? $hash_ref->{mountpoint} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{partition_label} = defined $hash_ref->{partlabel} ? $hash_ref->{partlabel} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{partition_uuid} = defined $hash_ref->{partuuid} ? $hash_ref->{partuuid} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{physical_sector_size} = defined $hash_ref->{'phy-sec'} ? $hash_ref->{'phy-sec'} : 0; + $anvil->{storage}{lsblk}{$kernel_device_name}{read_only} = defined $hash_ref->{ro} ? $hash_ref->{ro} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{removable} = defined $hash_ref->{rm} ? $hash_ref->{rm} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{rotating_drive} = defined $hash_ref->{rota} ? $hash_ref->{rota} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number} = defined $hash_ref->{serial} ? $hash_ref->{serial} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number} = $anvil->Words->clean_spaces({string => $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number}}); + $anvil->{storage}{lsblk}{$kernel_device_name}{scheduler} = defined $hash_ref->{sched} ? $hash_ref->{sched} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{size} = defined $hash_ref->{size} ? $hash_ref->{size} : 0; + $anvil->{storage}{lsblk}{$kernel_device_name}{'state'} = defined $hash_ref->{'state'} ? $hash_ref->{'state'} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{transport} = defined $hash_ref->{tran} ? $hash_ref->{tran} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{type} = $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_uuid} ? "partition" : "drive"; + $anvil->{storage}{lsblk}{$kernel_device_name}{vendor} = defined $hash_ref->{vendor} ? $hash_ref->{vendor} : ""; + $anvil->{storage}{lsblk}{$kernel_device_name}{vendor} = $anvil->Words->clean_spaces({string => $anvil->{storage}{lsblk}{$kernel_device_name}{vendor}}); + + # There's precious little data that comes from SD cards. + if ($kernel_device_name =~ /^mmcblk/) + { + if ($kernel_device_name =~ /^mmcblk\d+p\d+/) + { + # This is a partition + $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_type} = "partition"; + $anvil->{storage}{lsblk}{$kernel_device_name}{model} = "SD Card" if not $anvil->{storage}{lsblk}{$kernel_device_name}{model}; + $anvil->{storage}{lsblk}{$kernel_device_name}{transport} = "pci" if not $anvil->{storage}{lsblk}{$kernel_device_name}{transport}; + $anvil->{storage}{lsblk}{$kernel_device_name}{type} = "ssd" if not $anvil->{storage}{lsblk}{$kernel_device_name}{type}; + $anvil->{storage}{lsblk}{$kernel_device_name}{vendor} = "unknown" if not $anvil->{storage}{lsblk}{$kernel_device_name}{vendor}; + } + else + { + # It's the drive + $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_type} = "drive"; + } + } + # Later, we'll want to trace device mapper devices back to the real device behind them (being + # LVM, crypt, etc). For now, this works. + if ($kernel_device_name =~ /^dm-/) + { + $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_type} = "partition"; + $anvil->{storage}{lsblk}{$kernel_device_name}{model} = "Device Mapper" if not $anvil->{storage}{lsblk}{$kernel_device_name}{model}; + $anvil->{storage}{lsblk}{$kernel_device_name}{transport} = "virtual" if not $anvil->{storage}{lsblk}{$kernel_device_name}{transport}; + $anvil->{storage}{lsblk}{$kernel_device_name}{type} = "virtual" if not $anvil->{storage}{lsblk}{$kernel_device_name}{type}; + $anvil->{storage}{lsblk}{$kernel_device_name}{vendor} = "Linux" if not $anvil->{storage}{lsblk}{$kernel_device_name}{vendor}; + } + + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "storage::lsblk::${kernel_device_name}::alignment_offset" => $anvil->{storage}{lsblk}{$kernel_device_name}{alignment_offset}, + "storage::lsblk::${kernel_device_name}::device_type" => $anvil->{storage}{lsblk}{$kernel_device_name}{device_type}, + "storage::lsblk::${kernel_device_name}::filesystem_type" => $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_type}, + "storage::lsblk::${kernel_device_name}::filesystem_uuid" => $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_uuid}, + "storage::lsblk::${kernel_device_name}::hot_plug" => $anvil->{storage}{lsblk}{$kernel_device_name}{hot_plug}, + "storage::lsblk::${kernel_device_name}::logical_sector_size" => $anvil->{storage}{lsblk}{$kernel_device_name}{logical_sector_size}, + "storage::lsblk::${kernel_device_name}::model" => $anvil->{storage}{lsblk}{$kernel_device_name}{model}, + "storage::lsblk::${kernel_device_name}::mount_point" => $anvil->{storage}{lsblk}{$kernel_device_name}{mount_point}, + "storage::lsblk::${kernel_device_name}::partition_label" => $anvil->{storage}{lsblk}{$kernel_device_name}{partition_label}, + "storage::lsblk::${kernel_device_name}::partition_uuid" => $anvil->{storage}{lsblk}{$kernel_device_name}{partition_uuid}, + "storage::lsblk::${kernel_device_name}::physical_sector_size" => $anvil->{storage}{lsblk}{$kernel_device_name}{physical_sector_size}, + "storage::lsblk::${kernel_device_name}::read_only" => $anvil->{storage}{lsblk}{$kernel_device_name}{read_only}, + "storage::lsblk::${kernel_device_name}::removable" => $anvil->{storage}{lsblk}{$kernel_device_name}{removable}, + "storage::lsblk::${kernel_device_name}::rotating_drive" => $anvil->{storage}{lsblk}{$kernel_device_name}{rotating_drive}, + "storage::lsblk::${kernel_device_name}::serial_number" => $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number}, + "storage::lsblk::${kernel_device_name}::scheduler" => $anvil->{storage}{lsblk}{$kernel_device_name}{scheduler}, + "storage::lsblk::${kernel_device_name}::size" => $anvil->Convert->add_commas({number => $anvil->{storage}{lsblk}{$kernel_device_name}{size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->{storage}{lsblk}{$kernel_device_name}{size}}).")", + "storage::lsblk::${kernel_device_name}::state" => $anvil->{storage}{lsblk}{$kernel_device_name}{'state'}, + "storage::lsblk::${kernel_device_name}::type" => $anvil->{storage}{lsblk}{$kernel_device_name}{type}, + "storage::lsblk::${kernel_device_name}::transport" => $anvil->{storage}{lsblk}{$kernel_device_name}{transport}, + "storage::lsblk::${kernel_device_name}::vendor" => $anvil->{storage}{lsblk}{$kernel_device_name}{vendor}, + }}); + } + # Now loop through devices and pass parent information (like transport, model, etc) from devices down to partitions. + my $parent_device = ""; + foreach my $kernel_device_name (sort {$a cmp $b} keys %{$anvil->{storage}{lsblk}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { kernel_device_name => $kernel_device_name }}); + + if ($anvil->{storage}{lsblk}{$kernel_device_name}{type} eq "drive") + { + $parent_device = $kernel_device_name; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { parent_device => $parent_device }}); + next; + } + + if (($parent_device) && (not $anvil->{storage}{lsblk}{$kernel_device_name}{model})) + { + $anvil->{storage}{lsblk}{$kernel_device_name}{model} = $anvil->{storage}{lsblk}{$parent_device}{model}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "storage::lsblk::${kernel_device_name}::model" => $anvil->{storage}{lsblk}{$kernel_device_name}{model}, + }}); + } + if (($parent_device) && (not $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number})) + { + $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number} = $anvil->{storage}{lsblk}{$parent_device}{serial_number}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "storage::lsblk::${kernel_device_name}::serial_number" => $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number}, + }}); + } + if (($parent_device) && (not $anvil->{storage}{lsblk}{$kernel_device_name}{vendor})) + { + $anvil->{storage}{lsblk}{$kernel_device_name}{vendor} = $anvil->{storage}{lsblk}{$parent_device}{vendor}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "storage::lsblk::${kernel_device_name}::vendor" => $anvil->{storage}{lsblk}{$kernel_device_name}{vendor}, + }}); + } + if (($parent_device) && (not $anvil->{storage}{lsblk}{$kernel_device_name}{transport})) + { + $anvil->{storage}{lsblk}{$kernel_device_name}{transport} = $anvil->{storage}{lsblk}{$parent_device}{transport}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "storage::lsblk::${kernel_device_name}::transport" => $anvil->{storage}{lsblk}{$kernel_device_name}{transport}, + }}); + } + if (($parent_device) && (not $anvil->{storage}{lsblk}{$kernel_device_name}{'state'})) + { + $anvil->{storage}{lsblk}{$kernel_device_name}{'state'} = $anvil->{storage}{lsblk}{$parent_device}{'state'}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "storage::lsblk::${kernel_device_name}::state" => $anvil->{storage}{lsblk}{$kernel_device_name}{'state'}, + }}); + } + } + return(0); } diff --git a/Anvil/Tools/System.pm b/Anvil/Tools/System.pm index 961445b1..d060b2ae 100644 --- a/Anvil/Tools/System.pm +++ b/Anvil/Tools/System.pm @@ -36,6 +36,9 @@ my $THIS_FILE = "System.pm"; # maintenance_mode # manage_authorized_keys # manage_firewall +# parse_df +# parse_lsblk +# parse_lshw # read_ssh_config # reload_daemon # reboot_needed @@ -3806,6 +3809,151 @@ sub parse_arguments return($hash); } + +=head2 parse_lshw + +B<< NOTE >>: This method is not complete, do not use it yet! + +This calls C<< lshw >> (in XML format) and parses the output. Data is stored as: + + * lshw::... + +Parameters; + +=head3 xml (optional) + +If set, the passed-in XML is parsed and C<< lshw -xml >> is not called. This should only be used for testing / debugging. + +=cut +sub parse_lshw +{ + my $self = shift; + my $parameter = shift; + my $anvil = $self->parent; + my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "System->parse_lsblk()" }}); + + my $xml = defined $parameter->{xml} ? $parameter->{xml} : ""; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { xml => $xml }}); + + if (not $xml) + { + my $shell_call = $anvil->data->{path}{exe}{lshw}." -xml"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }}); + + ($xml, my $return_code) = $anvil->System->call({shell_call => $shell_call}); + if ($return_code) + { + # Failed. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "alert", key => "warning_0080", variables => { + return_code => $return_code, + output => $xml, + }}); + return(1); + } + } + + local $@; + my $dom = eval { XML::LibXML->load_xml(string => $xml); }; + if ($@) + { + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, key => "warning_0053", variables => { + cib => $xml, + error => $@, + }}); + } + + foreach my $node ($dom->findnodes('/list/node')) + { + my $id = $node->{id}; + my $class = $node->{class}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + id => $id, + class => $class, + }}); + } +=cut + foreach my $node ($dom->findnodes('/list/node')) + { + my $id = $node->{id}; + my $class = $node->{class}; + my $handle = $node->{handle}; + my $parent_description = defined $node->findvalue('./description') ? $node->findvalue('./description') : ""; + my $logical_name = defined $node->findvalue('./logicalname') ? $node->findvalue('./logicalname') : ""; + my $parent_vendor = defined $node->findvalue('./vendor') ? $node->findvalue('./vendor') : ""; + my $parent_model = defined $node->findvalue('./product') ? $node->findvalue('./product') : ""; + my $parent_serial_number = defined $node->findvalue('./serial') ? $node->findvalue('./serial') : ""; + my $parent_media = $id; + if ($id eq "device") + { + if (($parent_description =~ /sd card/i) or ($logical_name =~ /\/dev\/mmcblk/)) + { + $parent_media = "sdcard"; + } + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + id => $id, + class => $class, + logical_name => $logical_name, + parent_vendor => $parent_vendor, + parent_model => $parent_model, + }}); + # Sub devices may not appear, so we'll later match logical names (when they're a path) to devices in 'df' later. + foreach my $device ($node->findnodes('./node')) + { + my $dev_id = $device->{id}; + my $dev_class = $device->{class}; + my $bus_info = defined $device->findvalue('./businfo') ? $device->findvalue('./businfo') : ""; + my $path = defined $device->findvalue('./logicalname') ? $device->findvalue('./logicalname') : ""; + my $description = defined $device->findvalue('./description') ? $device->findvalue('./description') : ""; + my $vendor = defined $device->findvalue('./vendor') ? $device->findvalue('./vendor') : ""; + $vendor = $parent_vendor if not $vendor; + my $model = defined $device->findvalue('./product') ? $device->findvalue('./product') : ""; + $model = $parent_model if not $model; + my $serial_number = defined $device->findvalue('./serial') ? $device->findvalue('./serial') : ""; + $serial_number = $parent_serial_number if not $serial_number; + my $size_number = defined $device->findvalue('./size') ? $device->findvalue('./size') : ""; + $size_number = 0 if not $size_number; + my ($size_dom) = $device->findnodes('./size'); + my $size_units = $size_dom->{units}; + $size_units = "bytes" if not $size_units; + my $size_in_bytes = $anvil->Convert->human_readable_to_bytes({size => $size_number, type => $size_units}); + my $media = $dev_id; + if (($bus_info =~ /nvme/i) or ($path =~ /\/dev\/nvm/)) + { + $bus_info = "nvme"; + $media = "ssd"; + } + if ($dev_id eq "cdrom") + { + $media = "optical"; + } + if (($bus_info =~ /^scsi/) or ($description =~ /ATA Disk/i)) + { + # Call + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + dev_id => $dev_id, + dev_class => $dev_class, + bus_info => $bus_info, + path => $path, + description => $description, + vendor => $vendor, + model => $model, + serial_number => $serial_number, + size_number => $size_number, + size_units => $size_units, + size_in_bytes => $anvil->Convert->add_commas({number => $size_in_bytes})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $size_in_bytes}), + media => $media, + }}); + } + } +=cut + + return(0); +} + + =head2 read_ssh_config This reads /etc/ssh/ssh_config and notes hosts with defined ports. When found, the associated port will be automatically used for a given host name or IP address. diff --git a/scancore-agents/scan-filesystems/scan-filesystems b/scancore-agents/scan-filesystems/scan-filesystems index a9d80de1..a913ce33 100755 --- a/scancore-agents/scan-filesystems/scan-filesystems +++ b/scancore-agents/scan-filesystems/scan-filesystems @@ -107,16 +107,16 @@ sub find_changes =cut foreach my $scan_filesystem_type (keys %{$anvil->data->{filesystem}{scan_filesystem_type}}) { - my $scan_filesystem_device_path = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{name}; - my $scan_filesystem_mount_path = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{used_by_vg}; - my $scan_filesystem_media = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{attributes}; + my $scan_filesystem_kernel_name = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{name}; + my $scan_filesystem_mount_point = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{used_by_vg}; + my $scan_filesystem_media_type = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{attributes}; my $scan_filesystem_size = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{size}; my $scan_filesystem_free = $anvil->data->{filesystem}{scan_filesystem_type}{$scan_filesystem_type}{free_space}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { scan_filesystem_type => $scan_filesystem_type, - scan_filesystem_device_path => $scan_filesystem_device_path, - scan_filesystem_mount_path => $scan_filesystem_mount_path, - scan_filesystem_media => $scan_filesystem_media, + scan_filesystem_kernel_name => $scan_filesystem_kernel_name, + scan_filesystem_mount_point => $scan_filesystem_mount_point, + scan_filesystem_media_type => $scan_filesystem_media_type, scan_filesystem_size => $anvil->Convert->add_commas({number => $scan_filesystem_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_filesystem_size}), scan_filesystem_free => $anvil->Convert->add_commas({number => $scan_filesystem_free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $scan_filesystem_free}), }}); @@ -126,30 +126,30 @@ sub find_changes { # Yup, anything changed? my $scan_filesystem_uuid = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_uuid}; - my $old_scan_filesystem_device_path = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_device_path}; - my $old_scan_filesystem_mount_path = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_mount_path}; - my $old_scan_filesystem_media = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_media}; + my $old_scan_filesystem_kernel_name = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_kernel_name}; + my $old_scan_filesystem_mount_point = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_mount_point}; + my $old_scan_filesystem_media_type = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_media_type}; my $old_scan_filesystem_size = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_size}; my $old_scan_filesystem_free = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_free}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - old_scan_filesystem_device_path => $old_scan_filesystem_device_path, - old_scan_filesystem_mount_path => $old_scan_filesystem_mount_path, - old_scan_filesystem_media => $old_scan_filesystem_media, + old_scan_filesystem_kernel_name => $old_scan_filesystem_kernel_name, + old_scan_filesystem_mount_point => $old_scan_filesystem_mount_point, + old_scan_filesystem_media_type => $old_scan_filesystem_media_type, old_scan_filesystem_size => $anvil->Convert->add_commas({number => $old_scan_filesystem_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_scan_filesystem_size}), old_scan_filesystem_free => $anvil->Convert->add_commas({number => $old_scan_filesystem_free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $old_scan_filesystem_free}), }}); my $update = 0; - if ($scan_filesystem_device_path ne $old_scan_filesystem_device_path) + if ($scan_filesystem_kernel_name ne $old_scan_filesystem_kernel_name) { $update = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }}); - if ($old_scan_filesystem_device_path eq "DELETED") + if ($old_scan_filesystem_kernel_name eq "DELETED") { # A lost PV is back my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $scan_filesystem_device_path, + pv_name => $scan_filesystem_kernel_name, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_filesystem_alert_0001", variables => $variables}); $anvil->Alert->register({clear_alert => 1, alert_level => "warning", message => "scan_filesystem_alert_0001", variables => $variables, set_by => $THIS_FILE}); @@ -159,25 +159,25 @@ sub find_changes # Device (name) changed... This shouldn't normally happen. my $variables = { pv_uuid => $scan_filesystem_type, - new_name => $scan_filesystem_device_path, - old_name => $old_scan_filesystem_device_path, + new_name => $scan_filesystem_kernel_name, + old_name => $old_scan_filesystem_kernel_name, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_filesystem_alert_0002", variables => $variables}); $anvil->Alert->register({alert_level => "warning", message => "scan_filesystem_alert_0002", variables => $variables, set_by => $THIS_FILE}); } } - if ($scan_filesystem_mount_path ne $old_scan_filesystem_mount_path) + if ($scan_filesystem_mount_point ne $old_scan_filesystem_mount_point) { # If the old value was blank, then this PV was added to a VG and that's fine. $update = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }}); - if (not $old_scan_filesystem_mount_path) + if (not $old_scan_filesystem_mount_point) { # Added to a VG my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $scan_filesystem_device_path, - vg_name => $scan_filesystem_mount_path, + pv_name => $scan_filesystem_kernel_name, + vg_name => $scan_filesystem_mount_point, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_filesystem_alert_0004", variables => $variables}); $anvil->Alert->register({alert_level => "notice", message => "scan_filesystem_alert_0004", variables => $variables, set_by => $THIS_FILE}); @@ -187,24 +187,24 @@ sub find_changes # The VG was probably renamed. my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $scan_filesystem_device_path, - new_vg_name => $scan_filesystem_mount_path, - old_vg_name => $old_scan_filesystem_mount_path, + pv_name => $scan_filesystem_kernel_name, + new_vg_name => $scan_filesystem_mount_point, + old_vg_name => $old_scan_filesystem_mount_point, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_filesystem_alert_0005", variables => $variables}); $anvil->Alert->register({alert_level => "notice", message => "scan_filesystem_alert_0005", variables => $variables, set_by => $THIS_FILE}); } } - if ($scan_filesystem_media ne $old_scan_filesystem_media) + if ($scan_filesystem_media_type ne $old_scan_filesystem_media_type) { # Attribute bits changed. $update = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }}); my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $scan_filesystem_device_path, - new_attributes => $scan_filesystem_media, - old_attributes => $old_scan_filesystem_media, + pv_name => $scan_filesystem_kernel_name, + new_attributes => $scan_filesystem_media_type, + old_attributes => $old_scan_filesystem_media_type, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_filesystem_alert_0006", variables => $variables}); $anvil->Alert->register({alert_level => "notice", message => "scan_filesystem_alert_0006", variables => $variables, set_by => $THIS_FILE}); @@ -216,7 +216,7 @@ sub find_changes $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }}); my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $scan_filesystem_device_path, + pv_name => $scan_filesystem_kernel_name, new_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $scan_filesystem_size}), new_size_bytes => $anvil->Convert->add_commas({number => $scan_filesystem_size}), old_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_scan_filesystem_size}), @@ -242,7 +242,7 @@ sub find_changes $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update => $update }}); my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $scan_filesystem_device_path, + pv_name => $scan_filesystem_kernel_name, new_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $scan_filesystem_free}), new_free_bytes => $anvil->Convert->add_commas({number => $scan_filesystem_free}), old_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $old_scan_filesystem_free}), @@ -269,9 +269,9 @@ sub find_changes UPDATE scan_filesystems SET - scan_filesystem_device_path = ".$anvil->Database->quote($scan_filesystem_device_path).", - scan_filesystem_mount_path = ".$anvil->Database->quote($scan_filesystem_mount_path).", - scan_filesystem_media = ".$anvil->Database->quote($scan_filesystem_media).", + scan_filesystem_kernel_name = ".$anvil->Database->quote($scan_filesystem_kernel_name).", + scan_filesystem_mount_point = ".$anvil->Database->quote($scan_filesystem_mount_point).", + scan_filesystem_media_type = ".$anvil->Database->quote($scan_filesystem_media_type).", scan_filesystem_size = ".$anvil->Database->quote($scan_filesystem_size).", scan_filesystem_free = ".$anvil->Database->quote($scan_filesystem_free).", modified_date = ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." @@ -290,9 +290,9 @@ WHERE # New PV my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $scan_filesystem_device_path, - vg_name => $scan_filesystem_mount_path ? $scan_filesystem_mount_path : "--", - attributes => $scan_filesystem_media, + pv_name => $scan_filesystem_kernel_name, + vg_name => $scan_filesystem_mount_point ? $scan_filesystem_mount_point : "--", + attributes => $scan_filesystem_media_type, pv_size => $anvil->Convert->bytes_to_human_readable({'bytes' => $scan_filesystem_size}), pv_size_bytes => $anvil->Convert->add_commas({number => $scan_filesystem_size}), pv_free => $anvil->Convert->bytes_to_human_readable({'bytes' => $scan_filesystem_free}), @@ -309,9 +309,9 @@ INSERT INTO scan_filesystem_uuid, scan_filesystem_type, scan_filesystem_host_uuid, - scan_filesystem_device_path, - scan_filesystem_mount_path, - scan_filesystem_media, + scan_filesystem_kernel_name, + scan_filesystem_mount_point, + scan_filesystem_media_type, scan_filesystem_size, scan_filesystem_free, modified_date @@ -319,9 +319,9 @@ INSERT INTO ".$anvil->Database->quote($scan_filesystem_uuid).", ".$anvil->Database->quote($scan_filesystem_type).", ".$anvil->Database->quote($anvil->Get->host_uuid).", - ".$anvil->Database->quote($scan_filesystem_device_path).", - ".$anvil->Database->quote($scan_filesystem_mount_path).", - ".$anvil->Database->quote($scan_filesystem_media).", + ".$anvil->Database->quote($scan_filesystem_kernel_name).", + ".$anvil->Database->quote($scan_filesystem_mount_point).", + ".$anvil->Database->quote($scan_filesystem_media_type).", ".$anvil->Database->quote($scan_filesystem_size).", ".$anvil->Database->quote($scan_filesystem_free).", ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." @@ -336,18 +336,18 @@ INSERT INTO { # This one is missing. my $scan_filesystem_uuid = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_uuid}; - my $old_scan_filesystem_device_path = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_device_path}; + my $old_scan_filesystem_kernel_name = $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_kernel_name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:scan_filesystem_type' => $scan_filesystem_type, 's2:scan_filesystem_uuid' => $scan_filesystem_uuid, - 's3:old_scan_filesystem_device_path' => $old_scan_filesystem_device_path, + 's3:old_scan_filesystem_kernel_name' => $old_scan_filesystem_kernel_name, }}); - next if $old_scan_filesystem_device_path eq "DELETED"; + next if $old_scan_filesystem_kernel_name eq "DELETED"; # Register an alert my $variables = { pv_uuid => $scan_filesystem_type, - pv_name => $old_scan_filesystem_device_path, + pv_name => $old_scan_filesystem_kernel_name, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "scan_filesystem_alert_0003", variables => $variables}); $anvil->Alert->register({alert_level => "warning", message => "scan_filesystem_alert_0003", variables => $variables, set_by => $THIS_FILE}); @@ -357,7 +357,7 @@ INSERT INTO UPDATE scan_filesystems SET - scan_filesystem_device_path = 'DELETED', + scan_filesystem_kernel_name = 'DELETED', modified_date = ".$anvil->Database->quote($anvil->data->{sys}{database}{timestamp})." WHERE scan_filesystem_uuid = ".$anvil->Database->quote($scan_filesystem_uuid)." @@ -379,18 +379,22 @@ sub read_last_scan my $query = " SELECT scan_filesystem_uuid, + scan_filesystem_host_uuid, scan_filesystem_type, - scan_filesystem_device_path, - scan_filesystem_mount_path, - scan_filesystem_media, + scan_filesystem_kernel_name, + scan_filesystem_mount_point, + scan_filesystem_transport, + scan_filesystem_media_type, + scan_filesystem_vendor, + scan_filesystem_model, + scan_filesystem_serial_number, scan_filesystem_description, scan_filesystem_size, - scan_filesystem_free, - scan_filesystem_host_uuid + scan_filesystem_used FROM scan_filesystems ;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { query => $query }}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }}); my $results = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__}); my $count = @{$results}; @@ -402,24 +406,32 @@ FROM { # We've got an entry in the 'scan_filesystems' table, so now we'll look for data in the node and # services tables. - my $scan_filesystem_uuid = $row->[0]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_type} = $row->[1]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_device_path} = $row->[2]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_mount_path} = $row->[3]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_media} = $row->[4]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_description} = $row->[4]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_size} = $row->[5]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_free} = $row->[6]; - $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_host_uuid} = $row->[7]; + my $scan_filesystem_uuid = $row->[0]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_host_uuid} = $row->[1]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_type} = $row->[2]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_kernel_name} = $row->[3]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_mount_point} = $row->[4]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_transport} = $row->[5]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_media_type} = $row->[6]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_vendor} = $row->[7]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_model} = $row->[8]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_serial_number} = $row->[9]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_description} = $row->[10]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_size} = $row->[11]; + $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_used} = $row->[12]; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_uuid" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_uuid}, - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_device_path" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_device_path}, - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_mount_path" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_mount_path}, - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_media" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_media}, - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_description" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_description}, - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_size" => $anvil->Convert->add_commas({number => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_size}}), - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_free" => $anvil->Convert->add_commas({number => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_free}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_free}}), - "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_host_uuid" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_host_uuid}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_uuid" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_uuid}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_kernel_name" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_kernel_name}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_mount_point" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_mount_point}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_transport" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_transport}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_media_type" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_media_type}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_vendor" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_vendor}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_model" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_model}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_serial_number" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_serial_number}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_description" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_description}, + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_size" => $anvil->Convert->add_commas({number => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_size}}), + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_used" => $anvil->Convert->add_commas({number => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_used}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_used}}), + "sql::scan_filesystems::scan_filesystem_uuid::${scan_filesystem_uuid}::scan_filesystem_host_uuid" => $anvil->data->{sql}{scan_filesystems}{scan_filesystem_uuid}{$scan_filesystem_uuid}{scan_filesystem_host_uuid}, }}); } @@ -430,9 +442,68 @@ sub collect_data { my ($anvil) = @_; - ### TODO: Swap oput '--separator \#!\#' for '--reportformat json' - collect_bus_data($anvil); - collect_fs_data($anvil); + ### NOTE: If a drive is unmounted, we can't trust the sizes. + $anvil->Storage->parse_lsblk({debug => 2}); + $anvil->Storage->parse_df({debug => 2}); + foreach my $kernel_device_name (sort {$a cmp $b} keys %{$anvil->{storage}{lsblk}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { kernel_device_name => $kernel_device_name }}); + if ($anvil->{storage}{lsblk}{$kernel_device_name}{type} eq "partition") + { + $anvil->data->{new}{partition}{$kernel_device_name}{type} = $anvil->{storage}{lsblk}{$kernel_device_name}{filesystem_type}; + $anvil->data->{new}{partition}{$kernel_device_name}{mount_point} = $anvil->{storage}{lsblk}{$kernel_device_name}{mount_point}; + $anvil->data->{new}{partition}{$kernel_device_name}{transport} = $anvil->{storage}{lsblk}{$kernel_device_name}{transport}; + $anvil->data->{new}{partition}{$kernel_device_name}{media_type} = $anvil->{storage}{lsblk}{$kernel_device_name}{rotating_drive} ? "platter" : "solid_state"; # This could be network as well someday + $anvil->data->{new}{partition}{$kernel_device_name}{vendor} = $anvil->{storage}{lsblk}{$kernel_device_name}{vendor}; + $anvil->data->{new}{partition}{$kernel_device_name}{model} = $anvil->{storage}{lsblk}{$kernel_device_name}{model}; + $anvil->data->{new}{partition}{$kernel_device_name}{serial_number} = $anvil->{storage}{lsblk}{$kernel_device_name}{serial_number}; + $anvil->data->{new}{partition}{$kernel_device_name}{description} = $anvil->{storage}{lsblk}{$kernel_device_name}{partition_label}; + $anvil->data->{new}{partition}{$kernel_device_name}{size} = 0; + $anvil->data->{new}{partition}{$kernel_device_name}{used} = 0; + if (($anvil->data->{new}{partition}{$kernel_device_name}{mount_point}) && (exists $anvil->{storage}{df}{$kernel_device_name})) + { + # Look for space usage from 'df' + my $df_mount_point = $anvil->{storage}{df}{$kernel_device_name}{mount_point}; + my $df_filesystem_type = $anvil->{storage}{df}{$kernel_device_name}{filesystem_type}; + my $df_size = $anvil->{storage}{df}{$kernel_device_name}{size}; + my $df_used = $anvil->{storage}{df}{$kernel_device_name}{used}; + my $df_free = $anvil->{storage}{df}{$kernel_device_name}{free}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + df_filesystem_type => $df_filesystem_type, + df_mount_point => $df_mount_point, + size => $anvil->Convert->add_commas({number => $df_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $df_size}).")", + used => $anvil->Convert->add_commas({number => $df_used})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $df_used}).")", + free => $anvil->Convert->add_commas({number => $df_free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $df_free}).")", + }}); + + # This is a check to see if the calculated free space matches the reported free + # space. + $anvil->data->{new}{partition}{$kernel_device_name}{size} = $df_size; + $anvil->data->{new}{partition}{$kernel_device_name}{used} = $df_used; + my $calculated_free = $anvil->data->{new}{partition}{$kernel_device_name}{size} - $anvil->data->{new}{partition}{$kernel_device_name}{used}; + my $difference = $df_free - $calculated_free; + $difference =~ s/^-//; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + 's1:calculated_free' => $anvil->Convert->add_commas({number => $calculated_free})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $calculated_free}).")", + 's2:difference' => $anvil->Convert->add_commas({number => $difference})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $difference}).")", + }}); + } + + # Record the partition + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "s1:new::partition::${kernel_device_name}::type" => $anvil->data->{new}{partition}{$kernel_device_name}{type}, + "s2:new::partition::${kernel_device_name}::mount_point" => $anvil->data->{new}{partition}{$kernel_device_name}{mount_point}, + "s3:new::partition::${kernel_device_name}::transport" => $anvil->data->{new}{partition}{$kernel_device_name}{transport}, + "s4:new::partition::${kernel_device_name}::media_type" => $anvil->data->{new}{partition}{$kernel_device_name}{media_type}, + "s5:new::partition::${kernel_device_name}::vendor" => $anvil->data->{new}{partition}{$kernel_device_name}{vendor}, + "s6:new::partition::${kernel_device_name}::model" => $anvil->data->{new}{partition}{$kernel_device_name}{model}, + "s7:new::partition::${kernel_device_name}::serial_number" => $anvil->data->{new}{partition}{$kernel_device_name}{serial_number}, + "s8:new::partition::${kernel_device_name}::description" => $anvil->data->{new}{partition}{$kernel_device_name}{description}, + "s9:new::partition::${kernel_device_name}::size" => $anvil->Convert->add_commas({number => $anvil->data->{new}{partition}{$kernel_device_name}{size}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{new}{partition}{$kernel_device_name}{size}}).")", + "s10:new::partition::${kernel_device_name}::used" => $anvil->Convert->add_commas({number => $anvil->data->{new}{partition}{$kernel_device_name}{used}})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{new}{partition}{$kernel_device_name}{used}}).")", + }}); + } + } return(0); } @@ -494,7 +565,7 @@ sub collect_bus_data # { # next if $variable eq "id"; # $anvil->data->{cib}{parsed}{configuration}{crm_config}{cluster_property_set}{nvpair}{$nvpair_id}{$variable} = $nvpair->{$variable}; -# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { +# $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { # "cib::parsed::configuration::crm_config::cluster_property_set::nvpair::${nvpair_id}::${variable}" => $anvil->data->{cib}{parsed}{configuration}{crm_config}{cluster_property_set}{nvpair}{$nvpair_id}{$variable}, # }}); # } diff --git a/scancore-agents/scan-filesystems/scan-filesystems.sql b/scancore-agents/scan-filesystems/scan-filesystems.sql index 6b96601f..63dd1f23 100644 --- a/scancore-agents/scan-filesystems/scan-filesystems.sql +++ b/scancore-agents/scan-filesystems/scan-filesystems.sql @@ -6,16 +6,16 @@ CREATE TABLE scan_filesystems ( scan_filesystem_uuid uuid primary key, -- This comes from the file system's UUID scan_filesystem_host_uuid uuid not null, -- The host that the file system is mounted on. Note that some FSes, like those from USB, can move between hosts. scan_filesystem_type text not null, -- This is the name of the file system type. - scan_filesystem_device_path text not null, -- This is the backing device of the file system. - scan_filesystem_mount_path text not null, -- This is the name of the mount point. - scan_filesystem_media_type text not null, -- Optional description of the drive's media tpe (usb, nvme, sata_ssd, sata_platter, md, raid, optical, sdcard, etc - 'unknown') + scan_filesystem_kernel_name text not null, -- This is the backing device of the file system. + scan_filesystem_mount_point text not null, -- This is the name of the mount point. + scan_filesystem_transport text not null, -- Optional description of the drive's transport (usb, nvme, sata, sata, md, raid, optical, sdcard, etc - 'unknown') + scan_filesystem_media_type text not null, -- This is set to 'ssd' for solid state, 'platter' for spinning rust, 'network' for network mounts, etc. scan_filesystem_vendor text not null, -- Optional vendor of the drive the partition is on scan_filesystem_model text not null, -- Optional model of the drive the partiton is on scan_filesystem_serial_number text not null, -- Optional serial number of the drive the partition is on - scan_filesystem_removable text not null, -- Optional string indicating if the file system is on removable media ('yes', 'no', 'unknown') - scan_filesystem_description text not null, -- Free form description of the device. Usually make / model / serial number - scan_filesystem_size numeric not null, -- The size of the PV in bytes - scan_filesystem_free numeric not null, -- The free space, in bytes. + scan_filesystem_description text not null, -- Free form description of the device. + scan_filesystem_size numeric not null, -- The size of the partition, in bytes + scan_filesystem_used numeric not null, -- The used space, in bytes. modified_date timestamp with time zone not null, FOREIGN KEY(scan_filesystem_host_uuid) REFERENCES hosts(host_uuid) @@ -27,12 +27,16 @@ CREATE TABLE history.scan_filesystems ( scan_filesystem_uuid uuid, scan_filesystem_host_uuid uuid, scan_filesystem_type text, - scan_filesystem_device_path text, - scan_filesystem_mount_path text, - scan_filesystem_media text, + scan_filesystem_kernel_name text, + scan_filesystem_mount_point text, + scan_filesystem_transport text, + scan_filesystem_media_type text, + scan_filesystem_vendor text, + scan_filesystem_model text, + scan_filesystem_serial_number text, scan_filesystem_description text, scan_filesystem_size numeric, - scan_filesystem_free numeric, + scan_filesystem_used numeric, modified_date timestamp with time zone not null ); ALTER TABLE history.scan_filesystems OWNER TO admin; @@ -47,23 +51,31 @@ BEGIN (scan_filesystem_uuid, scan_filesystem_host_uuid, scan_filesystem_type, - scan_filesystem_device_path, - scan_filesystem_mount_path, - scan_filesystem_media, + scan_filesystem_kernel_name, + scan_filesystem_mount_point, + scan_filesystem_transport, + scan_filesystem_media_type, + scan_filesystem_vendor, + scan_filesystem_model, + scan_filesystem_serial_number, scan_filesystem_description, scan_filesystem_size, - scan_filesystem_free, + scan_filesystem_used, modified_date) VALUES (history_scan_filesystems.scan_filesystem_uuid, history_scan_filesystems.scan_filesystem_host_uuid, history_scan_filesystems.scan_filesystem_type, - history_scan_filesystems.scan_filesystem_device_path, - history_scan_filesystems.scan_filesystem_mount_path, - history_scan_filesystems.scan_filesystem_media, + history_scan_filesystems.scan_filesystem_kernel_name, + history_scan_filesystems.scan_filesystem_mount_point, + history_scan_filesystems.scan_filesystem_transport, + history_scan_filesystems.scan_filesystem_media_type, + history_scan_filesystems.scan_filesystem_vendor, + history_scan_filesystems.scan_filesystem_model, + history_scan_filesystems.scan_filesystem_serial_number, history_scan_filesystems.scan_filesystem_description, history_scan_filesystems.scan_filesystem_size, - history_scan_filesystems.scan_filesystem_free, + history_scan_filesystems.scan_filesystem_used, history_scan_filesystems.modified_date); RETURN NULL; END; diff --git a/scancore-agents/scan-hardware/scan-hardware b/scancore-agents/scan-hardware/scan-hardware index da1e88f0..6288b7fd 100755 --- a/scancore-agents/scan-hardware/scan-hardware +++ b/scancore-agents/scan-hardware/scan-hardware @@ -9,7 +9,7 @@ # 1 = Startup failure (not running as root, no DB, bad file read, etc) # # TODO: -# - +# - Finish System->parse_lwhw(), lots of good stuff in there. # use strict; diff --git a/share/words.xml b/share/words.xml index 6a8e83da..11bf252b 100644 --- a/share/words.xml +++ b/share/words.xml @@ -2319,6 +2319,8 @@ Read UUID: .... [#!variable!read_uuid!#] [ Warning ] - It looks like the postfix daemon is not running. Enabling and starting it now. [ Warning ] - Checking the mail queue after attempting to start postgres appears to have still failed. Output received was: [#!variable!output!#]. [ Warning ] - Not installing the Alteeve repo! The package: [#!variable!anvil_role_rpm!#] is already installed. This is OK, but be aware that updates from Alteeve will not be available. To change this, please install: [#!variable!alteeve_repo!#]. + [ Warning ] - Failed to read the JSON formatted output of 'lsblk'. Expected the return code '0' but received: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#]. + [ Warning ] - Failed to read the XML formatted output of 'lshw'. Expected the return code '0' but received: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].