From 0014cc591d312d6bfa2a57e475091d515fde6e37 Mon Sep 17 00:00:00 2001 From: digimer Date: Fri, 13 Oct 2023 16:58:35 -0400 Subject: [PATCH 1/2] Re-enabled DB connections in ocf:alteeve:server. Added DB connections to ocf:alteeve:server when starting or stopping servers. This is to ensure that the servers -> server_state are updated properly. Signed-off-by: digimer --- Anvil/Tools/Server.pm | 47 ++++++++++++++++++++++--------------------- ocf/alteeve/server | 14 +++++++++++++ 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/Anvil/Tools/Server.pm b/Anvil/Tools/Server.pm index 2fbd30f9..9d82781e 100644 --- a/Anvil/Tools/Server.pm +++ b/Anvil/Tools/Server.pm @@ -2635,29 +2635,27 @@ sub shutdown_virsh $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { "sys::database::connections" => $anvil->data->{sys}{database}{connections} }}); if ($anvil->data->{sys}{database}{connections}) { - if ($anvil->data->{sys}{database}{connections}) + my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({debug => $debug}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }}); + + $server_uuid = $anvil->Get->server_uuid_from_name({ + debug => $debug, + server_name => $server, + anvil_uuid => $anvil_uuid, + }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { server_uuid => $server_uuid }}); + if (($server_uuid) && ($server_uuid ne "!!error!!")) { - my $anvil_uuid = $anvil->Cluster->get_anvil_uuid({debug => $debug}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }}); - - $server_uuid = $anvil->Get->server_uuid_from_name({ - debug => $debug, - server_name => $server, - anvil_uuid => $anvil_uuid, - }); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { server_uuid => $server_uuid }}); - if (($server_uuid) && ($server_uuid ne "!!error!!")) + $anvil->Database->get_servers({debug => $debug}); + if (exists $anvil->data->{servers}{server_uuid}{$server_uuid}) { - $anvil->Database->get_servers({debug => $debug}); - if (exists $anvil->data->{servers}{server_uuid}{$server_uuid}) + my $old_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_state => $old_state }}); + + if ($old_state ne "in shutdown") { - my $old_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { old_state => $old_state }}); - - if ($old_state ne "in shutdown") - { - # Update it. - my $query = " + # Update it. + my $query = " UPDATE servers SET @@ -2666,9 +2664,8 @@ SET WHERE server_uuid = ".$anvil->Database->quote($server_uuid)." ;"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); - $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); - } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); + $anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__}); } } } @@ -2729,8 +2726,12 @@ WHERE }}); # Mark it as stopped now. (if we have a server_uuid, we have a database connection) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { server_uuid => $server_uuid }}); if ($server_uuid) { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { + "sys::database::connections" => $anvil->data->{sys}{database}{connections}, + }}); if ($anvil->data->{sys}{database}{connections}) { $anvil->Database->get_servers({debug => $debug}); diff --git a/ocf/alteeve/server b/ocf/alteeve/server index acdccb05..63776983 100755 --- a/ocf/alteeve/server +++ b/ocf/alteeve/server @@ -623,6 +623,13 @@ sub start_server # Start the resource, if needed. start_drbd_resource($anvil); + # Get a connection so that we can mark is as being in shutdown and shut off. + $anvil->Database->connect({ + check_for_resync => 0, + retry => 0, + sensitive => 1, + }); + # Still alive? Boot! my ($success) = $anvil->Server->boot_virsh({debug => 2, server => $server}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { success => $success }}); @@ -1174,6 +1181,13 @@ sub stop_server $anvil->System->check_storage(); $anvil->Server->get_status({debug => 2, server => $server}); + # Get a connection so that we can mark is as being in shutdown and shut off. + $anvil->Database->connect({ + check_for_resync => 0, + retry => 0, + sensitive => 1, + }); + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0313", variables => { server => $server }}); my $success = $anvil->Server->shutdown_virsh({server => $server}); if (not $success) From b1f89c272389ddd770829dcacef428cca28b2d15 Mon Sep 17 00:00:00 2001 From: digimer Date: Wed, 18 Oct 2023 18:17:49 -0400 Subject: [PATCH 2/2] Finished initial version of striker-show-jobs * Updated Database->get_jobs() to take 'job_host_uuid = all' to allow loading jobs from all cluster machines. Also updated it to record the 'job_host_uuid' and the unix timestamp version of 'modified_date'. Signed-off-by: digimer --- Anvil/Tools/Database.pm | 181 +++++++++------ man/Makefile.am | 1 + man/striker-boot-machine.8 | 2 +- man/striker-show-jobs.8 | 37 +++ scancore-agents/scan-server/scan-server | 10 +- share/words.xml | 3 + tools/Makefile.am | 1 + tools/striker-show-jobs | 287 ++++++++++++++++++++++++ 8 files changed, 445 insertions(+), 77 deletions(-) create mode 100644 man/striker-show-jobs.8 create mode 100755 tools/striker-show-jobs diff --git a/Anvil/Tools/Database.pm b/Anvil/Tools/Database.pm index 5e33a51e..718af7cc 100644 --- a/Anvil/Tools/Database.pm +++ b/Anvil/Tools/Database.pm @@ -4769,7 +4769,7 @@ Jobs that reached 100% within this number of seconds ago will be included. If th =head3 job_host_uuid (default $anvil->Get->host_uuid) -This is the host that we're getting a list of jobs from. +This is the host that we're getting a list of jobs from. If this is set to C<< all >>, all jobs are loaded from all hosts. This method takes no parameters. @@ -4782,13 +4782,25 @@ sub get_jobs my $debug = defined $parameter->{debug} ? $parameter->{debug} : 3; my $return = []; - my $ended_within = defined $parameter->{ended_within} ? $parameter->{ended_within} : 300; - my $job_host_uuid = defined $parameter->{job_host_uuid} ? $parameter->{job_host_uuid} : $anvil->Get->host_uuid; + my $ended_within = defined $parameter->{ended_within} ? $parameter->{ended_within} : 0; + my $job_host_uuid = defined $parameter->{job_host_uuid} ? $parameter->{job_host_uuid} : ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { ended_within => $ended_within, job_host_uuid => $job_host_uuid, }}); + if ($ended_within !~ /^\d+$/) + { + $ended_within = 300; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { ended_within => $ended_within }}); + } + + if (not $job_host_uuid) + { + $job_host_uuid = $anvil->Get->host_uuid; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { job_host_uuid => $job_host_uuid }}); + } + if (exists $anvil->data->{jobs}{running}) { delete $anvil->data->{jobs}{running}; @@ -4811,11 +4823,18 @@ SELECT job_title, job_description, job_status, - modified_date + job_host_uuid, + modified_date, + round(extract(epoch from modified_date)) FROM - jobs + jobs "; + if ($job_host_uuid ne "all") + { + $query .= " WHERE - job_host_uuid = ".$anvil->Database->quote($job_host_uuid)." + job_host_uuid = ".$anvil->Database->quote($job_host_uuid); + } + $query .= " ;"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }}); @@ -4838,7 +4857,9 @@ WHERE my $job_title = $row->[8]; my $job_description = $row->[9]; my $job_status = $row->[10]; - my $modified_date = $row->[11]; + my $job_host_uuid = $row->[11]; + my $modified_date = $row->[12]; + my $modified_date_unix = $row->[13]; my $now_time = time; my $started_seconds_ago = $job_picked_up_at ? ($now_time - $job_picked_up_at) : 0; my $updated_seconds_ago = $job_updated ? ($now_time - $job_updated) : 0; @@ -4853,8 +4874,10 @@ WHERE job_progress => $job_progress, job_title => $job_title, job_description => $job_description, + job_host_uuid => $job_host_uuid, job_status => $job_status, modified_date => $modified_date, + modified_date_unix => $modified_date_unix, now_time => $now_time, started_seconds_ago => $started_seconds_ago, updated_seconds_ago => $updated_seconds_ago, @@ -4868,67 +4891,77 @@ WHERE } push @{$return}, { - job_uuid => $job_uuid, - job_command => $job_command, - job_data => $job_data, - job_picked_up_by => $job_picked_up_by, - job_picked_up_at => $job_picked_up_at, - job_updated => $job_updated, - job_name => $job_name, - job_progress => $job_progress, - job_title => $job_title, - job_description => $job_description, - job_status => $job_status, - modified_date => $modified_date, + job_uuid => $job_uuid, + job_command => $job_command, + job_data => $job_data, + job_picked_up_by => $job_picked_up_by, + job_picked_up_at => $job_picked_up_at, + job_updated => $job_updated, + job_name => $job_name, + job_progress => $job_progress, + job_title => $job_title, + job_description => $job_description, + job_status => $job_status, + job_host_uuid => $job_host_uuid, + modified_date => $modified_date, + modified_date_unix => $modified_date_unix, }; - $anvil->data->{jobs}{running}{$job_uuid}{job_command} = $job_command; - $anvil->data->{jobs}{running}{$job_uuid}{job_data} = $job_data; - $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_by} = $job_picked_up_by; - $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_at} = $job_picked_up_at; - $anvil->data->{jobs}{running}{$job_uuid}{job_updated} = $job_updated; - $anvil->data->{jobs}{running}{$job_uuid}{job_name} = $job_name; - $anvil->data->{jobs}{running}{$job_uuid}{job_progress} = $job_progress; - $anvil->data->{jobs}{running}{$job_uuid}{job_title} = $job_title; - $anvil->data->{jobs}{running}{$job_uuid}{job_description} = $job_description; - $anvil->data->{jobs}{running}{$job_uuid}{job_status} = $job_status; - $anvil->data->{jobs}{running}{$job_uuid}{modified_date} = $modified_date; + $anvil->data->{jobs}{running}{$job_uuid}{job_command} = $job_command; + $anvil->data->{jobs}{running}{$job_uuid}{job_data} = $job_data; + $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_by} = $job_picked_up_by; + $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_at} = $job_picked_up_at; + $anvil->data->{jobs}{running}{$job_uuid}{job_updated} = $job_updated; + $anvil->data->{jobs}{running}{$job_uuid}{job_name} = $job_name; + $anvil->data->{jobs}{running}{$job_uuid}{job_progress} = $job_progress; + $anvil->data->{jobs}{running}{$job_uuid}{job_title} = $job_title; + $anvil->data->{jobs}{running}{$job_uuid}{job_description} = $job_description; + $anvil->data->{jobs}{running}{$job_uuid}{job_status} = $job_status; + $anvil->data->{jobs}{running}{$job_uuid}{job_host_uuid} = $job_host_uuid; + $anvil->data->{jobs}{running}{$job_uuid}{modified_date} = $modified_date; + $anvil->data->{jobs}{running}{$job_uuid}{modified_date_unix} = $modified_date_unix; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "jobs::running::${job_uuid}::job_command" => $anvil->data->{jobs}{running}{$job_uuid}{job_command}, - "jobs::running::${job_uuid}::job_data" => $anvil->data->{jobs}{running}{$job_uuid}{job_data}, - "jobs::running::${job_uuid}::job_picked_up_by" => $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_by}, - "jobs::running::${job_uuid}::job_picked_up_at" => $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_at}, - "jobs::running::${job_uuid}::job_updated" => $anvil->data->{jobs}{running}{$job_uuid}{job_updated}, - "jobs::running::${job_uuid}::job_name" => $anvil->data->{jobs}{running}{$job_uuid}{job_name}, - "jobs::running::${job_uuid}::job_progress" => $anvil->data->{jobs}{running}{$job_uuid}{job_progress}, - "jobs::running::${job_uuid}::job_title" => $anvil->data->{jobs}{running}{$job_uuid}{job_title}, - "jobs::running::${job_uuid}::job_description" => $anvil->data->{jobs}{running}{$job_uuid}{job_description}, - "jobs::running::${job_uuid}::job_status" => $anvil->data->{jobs}{running}{$job_uuid}{job_status}, - "jobs::running::${job_uuid}::modified_date" => $anvil->data->{jobs}{running}{$job_uuid}{modified_date}, + "jobs::running::${job_uuid}::job_command" => $anvil->data->{jobs}{running}{$job_uuid}{job_command}, + "jobs::running::${job_uuid}::job_data" => $anvil->data->{jobs}{running}{$job_uuid}{job_data}, + "jobs::running::${job_uuid}::job_picked_up_by" => $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_by}, + "jobs::running::${job_uuid}::job_picked_up_at" => $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_at}, + "jobs::running::${job_uuid}::job_updated" => $anvil->data->{jobs}{running}{$job_uuid}{job_updated}, + "jobs::running::${job_uuid}::job_name" => $anvil->data->{jobs}{running}{$job_uuid}{job_name}, + "jobs::running::${job_uuid}::job_progress" => $anvil->data->{jobs}{running}{$job_uuid}{job_progress}, + "jobs::running::${job_uuid}::job_title" => $anvil->data->{jobs}{running}{$job_uuid}{job_title}, + "jobs::running::${job_uuid}::job_description" => $anvil->data->{jobs}{running}{$job_uuid}{job_description}, + "jobs::running::${job_uuid}::job_status" => $anvil->data->{jobs}{running}{$job_uuid}{job_status}, + "jobs::running::${job_uuid}::job_host_uuid" => $anvil->data->{jobs}{running}{$job_uuid}{job_host_uuid}, + "jobs::running::${job_uuid}::modified_date" => $anvil->data->{jobs}{running}{$job_uuid}{modified_date}, + "jobs::running::${job_uuid}::modified_date_unix" => $anvil->data->{jobs}{running}{$job_uuid}{modified_date}, }}); # Make it possible to sort by modified date for serial execution of similar jobs. - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_command} = $job_command; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_data} = $job_data; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_by} = $job_picked_up_by; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_at} = $job_picked_up_at; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_updated} = $job_updated; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_name} = $job_name; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_progress} = $job_progress; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_title} = $job_title; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_description} = $job_description; - $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_status} = $job_status; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_command} = $job_command; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_data} = $job_data; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_by} = $job_picked_up_by; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_at} = $job_picked_up_at; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_updated} = $job_updated; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_name} = $job_name; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_progress} = $job_progress; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_title} = $job_title; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_description} = $job_description; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_status} = $job_status; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_host_uuid} = $job_host_uuid; + $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{modified_date_unix} = $modified_date_unix; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_command" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_command}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_data" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_data}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_picked_up_by" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_by}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_picked_up_at" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_at}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_updated" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_updated}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_name" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_name}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_progress" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_progress}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_title" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_title}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_description" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_description}, - "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_status" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_status}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_command" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_command}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_data" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_data}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_picked_up_by" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_by}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_picked_up_at" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_picked_up_at}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_updated" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_updated}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_name" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_name}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_progress" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_progress}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_title" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_title}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_description" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_description}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_status" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_status}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::job_host_uuid" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{job_host_uuid}, + "jobs::modified_date::${modified_date}::job_uuid::${job_uuid}::modified_date_unix" => $anvil->data->{jobs}{modified_date}{$modified_date}{job_uuid}{$job_uuid}{modified_date_unix}, }}); } @@ -4940,18 +4973,20 @@ WHERE foreach my $hash_ref (@{$return}) { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { - job_uuid => $hash_ref->{job_uuid}, - job_command => $hash_ref->{job_command}, - job_data => $hash_ref->{job_data}, - job_picked_up_by => $hash_ref->{job_picked_up_by}, - job_picked_up_at => $hash_ref->{job_picked_up_at}, - job_updated => $hash_ref->{job_updated}, - job_name => $hash_ref->{job_name}, - job_progress => $hash_ref->{job_progress}, - job_title => $hash_ref->{job_title}, - job_description => $hash_ref->{job_description}, - job_status => $hash_ref->{job_status}, - modified_date => $hash_ref->{modified_date}, + job_uuid => $hash_ref->{job_uuid}, + job_command => $hash_ref->{job_command}, + job_data => $hash_ref->{job_data}, + job_picked_up_by => $hash_ref->{job_picked_up_by}, + job_picked_up_at => $hash_ref->{job_picked_up_at}, + job_updated => $hash_ref->{job_updated}, + job_name => $hash_ref->{job_name}, + job_progress => $hash_ref->{job_progress}, + job_title => $hash_ref->{job_title}, + job_description => $hash_ref->{job_description}, + job_status => $hash_ref->{job_status}, + job_host_uuid => $hash_ref->{job_host_uuid}, + modified_date => $hash_ref->{modified_date}, + modified_date_unix => $hash_ref->{modified_date_unix}, }}); } } diff --git a/man/Makefile.am b/man/Makefile.am index effeaebd..44a81a00 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -71,6 +71,7 @@ dist_man8_MANS = \ striker-purge-target.8 \ striker-scan-network.8 \ striker-show-db-counts.8 \ + striker-show-jobs.8 \ striker-update-cluster.8 \ tool-fio-tester.8 \ unfence_pacemaker.8 diff --git a/man/striker-boot-machine.8 b/man/striker-boot-machine.8 index 1393aac3..32f90945 100644 --- a/man/striker-boot-machine.8 +++ b/man/striker-boot-machine.8 @@ -1,6 +1,6 @@ .\" Manpage for the Anvil! server boot program .\" Contact mkelly@alteeve.com to report issues, concerns or suggestions. -.TH striker-boot-machine "8" "Octobober 12 2023" "Anvil! Intelligent Availability™ Platform" +.TH striker-boot-machine "8" "October 12 2023" "Anvil! Intelligent Availability™ Platform" .SH NAME striker-boot-machine \- Tool used to boot physical machines that have IPMI configuration information. .SH SYNOPSIS diff --git a/man/striker-show-jobs.8 b/man/striker-show-jobs.8 new file mode 100644 index 00000000..f202991d --- /dev/null +++ b/man/striker-show-jobs.8 @@ -0,0 +1,37 @@ +.\" Manpage for the Anvil! server boot program +.\" Contact mkelly@alteeve.com to report issues, concerns or suggestions. +.TH striker-show-jobs "8" "October 18 2023" "Anvil! Intelligent Availability™ Platform" +.SH NAME +striker-show-jobs \- This shows the queued, running and (recently) completed jobs. +.SH SYNOPSIS +.B striker-show-jobs +\fI\, \/\fR[\fI\,options\/\fR] +.SH DESCRIPTION +This shows information about jobs that are queued, running or completed within a set amount of time on the command line. +.TP +.SH OPTIONS +.TP +\-?, \-h, \fB\-\-help\fR +Show this man page. +.TP +\fB\-\-log-secure\fR +When logging, record sensitive data, like passwords. +.TP +\-v, \-vv, \-vvv +Set the log level to 1, 2 or 3 respectively. Be aware that level 3 generates a significant amount of log data. +.SS "Commands:" +.TP +\fB\-\-ended\-within\fR +This allows viewing jobs that have already completed. This is the number of seconds (default '300') back in time that you want to see completed jobs. Jobs that completed more than this amount of time ago will be ignored. +.TP +\fB\-\-host\fR +This is the host name or UUID that you want to see jobs from. The default is 'all' and shows all machines in the Anvil! cluster. +.TP +Setting this to '0' will hide completed jobs. +\fB\-\-job\-uuid\fR +This allows showing the details of only one specific job. +.IP +.SH AUTHOR +Written by Madison Kelly, Alteeve staff and the Anvil! project contributors. +.SH "REPORTING BUGS" +Report bugs to users@clusterlabs.org diff --git a/scancore-agents/scan-server/scan-server b/scancore-agents/scan-server/scan-server index f0b30d2f..939cf4a7 100755 --- a/scancore-agents/scan-server/scan-server +++ b/scancore-agents/scan-server/scan-server @@ -12,8 +12,7 @@ # 2 = libvirtd is not running. # # BUG: -# - Check that an update on disk is not overwritten by the old config still being in memory for a still- -# running VM (specifically, RAM updates) +# - Servers that are off, but not marked as such in the DB, needs to be updated. (See: line ~1105) # # TODO: # - Move location constraints to the host node if the server is not on the preferred host (this happens after @@ -436,7 +435,12 @@ sub collect_data # Get the server state. my $server_state = get_server_state($anvil, $server_name); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_state => $server_state }}); - + + $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state} = $server_state; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "scan-server::server_name::${server_name}::server_state" => $anvil->data->{'scan-server'}{server_name}{$server_name}{server_state}, + }}); + # This is set to 1 when we add a server, supressing detailed change checks as we'll have # already registered an alert. my $added = 0; diff --git a/share/words.xml b/share/words.xml index 83f46d16..0ee0c687 100644 --- a/share/words.xml +++ b/share/words.xml @@ -1157,6 +1157,9 @@ resource #!variable!server!# { Estimated Runtime Last Updated Optical Disc + Queued Jobs + Jobs In Progress + Completed Jobs Configure Network diff --git a/tools/Makefile.am b/tools/Makefile.am index 4533abc0..200d0145 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -68,6 +68,7 @@ dist_sbin_SCRIPTS = \ striker-purge-target \ striker-scan-network \ striker-show-db-counts \ + striker-show-jobs \ striker-update-cluster fencedir = ${FASEXECPREFIX}/sbin diff --git a/tools/striker-show-jobs b/tools/striker-show-jobs new file mode 100755 index 00000000..b9286d2c --- /dev/null +++ b/tools/striker-show-jobs @@ -0,0 +1,287 @@ +#!/usr/bin/perl +# +# This program will boot a target machine using either it's IPMI interface, if available, or one of the +# (non-PDU) fence methods, if the target is in an Anvil! and we have a manifest for it. +# +# Exit codes; +# 0 = Normal exit. +# 1 = No database connection. +# +# TODO: +# + +use strict; +use warnings; +use Anvil::Tools; + +my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; +my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; +if (($running_directory =~ /^\./) && ($ENV{PWD})) +{ + $running_directory =~ s/^\./$ENV{PWD}/; +} + +# Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. +$| = 1; + +my $anvil = Anvil::Tools->new(); + +$anvil->Get->switches({list => [ + "ended-within", + "host", + "job-uuid"], man => $THIS_FILE}); +$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); + +# Make sure we've got a +if (($anvil->data->{switches}{'ended-within'} eq "") or ($anvil->data->{switches}{'ended-within'} !~ /^\d+$/)) +{ + $anvil->data->{switches}{'ended-within'} = 300; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "switches::ended-within" => $anvil->data->{switches}{'ended-within'}, + }}); +} + +$anvil->data->{show}{host_uuid} = ""; +if (($anvil->data->{switches}{host}) && ($anvil->data->{switches}{host} ne "all")) +{ + # Get the host_uuid + $anvil->data->{show}{host_uuid} = $anvil->Database->get_host_uuid_from_string({debug => 2, string => $anvil->data->{switches}{host}}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "show::host_uuid" => $anvil->data->{show}{host_uuid}, + }}); +} + +$anvil->Database->connect(); +$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); +if (not $anvil->data->{sys}{database}{connections}) +{ + # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try + # again after we exit. + $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"}); + sleep 10; + $anvil->nice_exit({exit_code => 1}); +} + +show_jobs($anvil); + +$anvil->nice_exit({exit_code => 0}); + + +############################################################################################################# +# Functions # +############################################################################################################# + +sub show_jobs +{ + my ($anvil) = @_; + + $anvil->Database->get_hosts({debug => 3}); + $anvil->Database->get_jobs({ + debug => 2, + job_host_uuid => $anvil->data->{show}{host_uuid} // "all", + ended_within => $anvil->data->{switches}{'ended-within'}, + }); + + # Sort them into hosts, and then sort them by picked-up time (if we sorted by modified date, the + # display would jump all over the place) + # NOTE: Yes, I know, sorting on job_uuid doesn't make much sense but it means repeated runs are + # at least consistent in the logss. + my $jobs_found = 0; + foreach my $job_uuid (sort {$a cmp $b} keys %{$anvil->data->{jobs}{running}}) + { + if (($anvil->data->{switches}{'job-uuid'}) && ($anvil->data->{switches}{'job-uuid'} ne $job_uuid)) + { + next; + } + $jobs_found++; + my $job_command = $anvil->data->{jobs}{running}{$job_uuid}{job_command}; + my $job_data = $anvil->data->{jobs}{running}{$job_uuid}{job_data}; + my $job_picked_up_by = $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_by}; + my $job_picked_up_at = $anvil->data->{jobs}{running}{$job_uuid}{job_picked_up_at}; + my $job_updated = $anvil->data->{jobs}{running}{$job_uuid}{job_updated}; + my $job_name = $anvil->data->{jobs}{running}{$job_uuid}{job_name}; + my $job_progress = $anvil->data->{jobs}{running}{$job_uuid}{job_progress}; + my $job_title = $anvil->data->{jobs}{running}{$job_uuid}{job_title}; + my $job_description = $anvil->data->{jobs}{running}{$job_uuid}{job_description}; + my $job_status = $anvil->data->{jobs}{running}{$job_uuid}{job_status}; + my $job_host_uuid = $anvil->data->{jobs}{running}{$job_uuid}{job_host_uuid}; + my $short_host_name = $anvil->data->{hosts}{host_uuid}{$job_host_uuid}{short_host_name} // ""; + my $modified_date = $anvil->data->{jobs}{running}{$job_uuid}{modified_date}; + my $modified_date_unix = $anvil->data->{jobs}{running}{$job_uuid}{modified_date_unix}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + jobs_found => $jobs_found, + job_command => $job_command, + job_data => $job_data, + job_picked_up_by => $job_picked_up_by, + job_picked_up_at => $job_picked_up_at, + job_updated => $job_updated, + job_name => $job_name, + job_progress => $job_progress, + job_title => $job_title, + job_description => $job_description, + job_status => $job_status, + job_host_uuid => $job_host_uuid, + short_host_name => $short_host_name, + modified_date => $modified_date, + modified_date_unix => $modified_date_unix, + }}); + + # Turn the time stamps into strings and translate the job_title, job_description, and job_status + my $say_job_picked_up_at = $anvil->Get->date_and_time({use_time => $job_picked_up_at}); + my $say_job_updated = $anvil->Get->date_and_time({use_time => $job_updated}); + my $say_job_title = $anvil->Words->parse_banged_string({debug => 2, key_string => $job_title}); + my $say_job_description = $anvil->Words->parse_banged_string({debug => 2, key_string => $job_description}); + my $say_job_status = $anvil->Words->parse_banged_string({debug => 2, key_string => $job_status}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + say_job_picked_up_at => $say_job_picked_up_at, + say_job_updated => $say_job_updated, + say_job_title => $say_job_title, + say_job_description => $say_job_description, + say_job_status => $say_job_status, + }}); + + my $sort_key = "queued"; + if ($job_progress == 100) + { + # Finished + $sort_key = "completed"; + } + elsif ($job_progress) + { + # Picked up + $sort_key = "in_progress"; + } + + if (not exists $anvil->data->{count}{$short_host_name}{$sort_key}) + { + $anvil->data->{count}{$short_host_name}{$sort_key} = 1; + } + else + { + $anvil->data->{count}{$short_host_name}{$sort_key}++; + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "count::${short_host_name}::${sort_key}" => $anvil->data->{count}{$short_host_name}{$sort_key}, + }}); + + # Store + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_uuid} = $job_uuid; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_command} = $job_command; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_data} = $job_data; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_picked_up_by} = $job_picked_up_by; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_updated} = $say_job_updated; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_picked_up_at} = $say_job_picked_up_at; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_name} = $job_name; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_progress} = $job_progress; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_title} = $say_job_title; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_description} = $say_job_description; + $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_status} = $say_job_status; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_uuid" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_uuid}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_command" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_command}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_data" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_data}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_picked_up_by" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_picked_up_by}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_updated" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_updated}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_picked_up_at" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_picked_up_at}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_name" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_name}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_progress" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_progress}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_title" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_title}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_description" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_description}, + "sorted_jobs::${short_host_name}::${sort_key}::${job_picked_up_at}::job_status" => $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_status}, + }}); + } + + if (($anvil->data->{switches}{'job-uuid'}) && (not $jobs_found)) + { + print "The job: [".$anvil->data->{switches}{'job-uuid'}."] was not found.\n"; + $anvil->nice_exit({exit_code => 1}); + } + + foreach my $short_host_name (sort {$a cmp $b} keys %{$anvil->data->{sorted_jobs}}) + { + print "-=] Jobs on ".$short_host_name." [=------------------------------------------------\n"; + foreach my $sort_key ("queued", "in_progress", "completed") + { + my $count = exists $anvil->data->{count}{$short_host_name}{$sort_key} // 0; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + sort_key => $sort_key, + count => $count, + }}); + next if not $count; + + my $say_header = ""; + if ($sort_key eq "queued") + { + $say_header = $anvil->Words->string({key => 'header_0112'}); + } + elsif ($sort_key eq "in_progress") + { + $say_header = $anvil->Words->string({key => 'header_0113'}); + } + elsif ($sort_key eq "completed") + { + $say_header = $anvil->Words->string({key => 'header_0114'}); + } + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_header => $say_header }}); + print "- ".$say_header."\n"; + foreach my $job_picked_up_at (sort {$a cmp $b} keys %{$anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}}) + { + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_picked_up_at => $job_picked_up_at }}); + my $job_uuid = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_uuid}; + my $job_command = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_command}; + my $job_data = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_data}; + my $job_picked_up_by = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_picked_up_by}; + my $say_job_updated = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_updated}; + my $say_job_picked_up_at = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_picked_up_at}; + my $job_name = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_name}; + my $job_progress = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_progress}; + my $say_job_title = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_title}; + my $say_job_description = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_description}; + my $say_job_status = $anvil->data->{sorted_jobs}{$short_host_name}{$sort_key}{$job_picked_up_at}{job_status}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + job_uuid => $job_uuid, + job_command => $job_command, + job_data => $job_data, + job_picked_up_by => $job_picked_up_by, + say_job_updated => $say_job_updated, + say_job_picked_up_at => $say_job_picked_up_at, + job_name => $job_name, + job_progress => $job_progress, + say_job_title => $say_job_title, + say_job_description => $say_job_description, + say_job_status => $say_job_status, + }}); + print "Job Name: [".$job_name."], UUID: [".$job_uuid."]\n"; + if ($sort_key eq "queued") + { + print "- Command: [".$job_command."], \n"; + } + elsif ($sort_key eq "completed") + { + print "- Command: [".$job_command."], Picked up at: [".$say_job_picked_up_at."], last updated: [".$say_job_updated."]\n"; + } + else + { + print "- Command: [".$job_command."], Progress: [".$job_progress."\%] Picked up at: [".$say_job_picked_up_at."], last updated: [".$say_job_updated."], PID: [".$job_picked_up_by."]\n"; + } + my $job_data_lines = (split/\n/, $job_data); + if ($job_data_lines > 1) + { + print "---] Job Data:\n"; + print $job_data."\n"; + print "----------------\n"; + } + else + { + print "- Job Data: [".$job_data."]\n"; + } + print "---] Job Status:\n"; + print $say_job_status."\n"; + print "--------------------------------------------------------------------------------\n"; + } + } + } + + return(0); +}