seasharp/anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
eeab34059b
anvil/tools/anvil-update-system
digimer ed480cf1cb * Fixed a double-$ bug in Remote->_check_known_hosts_for_target() 
* Updated striker-update-cluster to take '--timeout' and a number of seconds, or 'Xm' or 'Xh' for minutes or hourse, respectively. Also updated to show the remaining time while waiting, and added waiting timeout to the rest of the while loops that prior had no time limit. This addresses issue #383 and issue #382.

Signed-off-by: digimer <mkelly@alteeve.ca>
2023-07-25 19:13:41 -04:00

622 lines
23 KiB
Perl
Executable File
Raw Blame History

#!/usr/bin/perl
#
# This updates the host system.
# NOTE: This doesn't update the Anvil! software stack yet, just the OS.
#
# - On Striker; This will take the system offline and then run 'dnf -y update'.
# - On Nodes; This will do nothing until all servers are off the node. Then the node will be withdrawn,
# updated and then rejoin the cluster.
# - On DR; This will do nothing until no servers are running, then it will update the system.
#
# In all cases, the system will be rebooted if the kernel is updated.
#
# Exit codes;
# 0 = Normal exit.
# 1 = No database connections available.
# 2 = The job UUID was passed, but it wasn't valid.
# 3 = It looks like the update failed, reset progress to '0'.
# 4 = Failed to withdraw the node from the cluster.
#
# TODO:
# - Rebuild this to be 'striker-update-system' and have it update local strikers and all nodes.
#
use strict;
use warnings;
use Anvil::Tools;
# Disable buffering
$| = 1;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
if (($running_directory =~ /^\./) && ($ENV{PWD}))
{
$running_directory =~ s/^\./$ENV{PWD}/;
}
my $anvil = Anvil::Tools->new();
### TODO: Remove this before final release
$anvil->Log->level({set => 2});
$anvil->Log->secure({set => 1});
##########################################
# Read switches (target ([user@]host[:port]) and the file with the target's password.
$anvil->Get->switches({list => [
"clear-cache",
"no-db",
"no-reboot",
"reboot"], man => $THIS_FILE});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
# Connect to DBs.
if ($anvil->data->{switches}{'no-db'})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "log_0743"});
# If there was a job-uuid, clear it.
$anvil->data->{sys}{database}{connections} = 0;
$anvil->data->{switches}{'job-uuid'} = "";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'sys::database::connections' => $anvil->data->{sys}{database}{connections},
'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'},
}});
}
else
{
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections})
{
# No databases, exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, secure => 0, key => "error_0003"});
$anvil->nice_exit({exit_code => 1});
}
}
$anvil->data->{jobs}{job_uuid} = "";
if ($anvil->data->{switches}{'job-uuid'})
{
# See if another instance is running. If so, sleep for 10 seconds and then exit. The other instance
# could be the '--no-db' run we're about to clobber.
my $pids = $anvil->System->pids({
ignore_me => 1,
program_name => $THIS_FILE,
});
my $other_instances = @{$pids};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { other_instances => $other_instances }});
if ($other_instances)
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "message_0233", variables => { program => $THIS_FILE }});
sleep 10;
$anvil->nice_exit({exit_code => 0});
}
# Load the job details. If anything is returned, there was a problem.
my $return = $anvil->Job->get_job_details({job_uuid => $anvil->data->{switches}{'job-uuid'}});
if ($return)
{
# It's not a UUID.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 1, list => { 'return' => $return }});
$anvil->nice_exit({exit_code => 2});
}
}
# Clear any old runs.
update_progress($anvil, 0, "clear");
# We'll keep a count of lines and packages to show the user.
$anvil->data->{counts}{downloaded} = 0;
$anvil->data->{counts}{installed} = 0;
$anvil->data->{counts}{verified} = 0;
$anvil->data->{counts}{lines} = 0;
# Mark that we're starting
update_progress($anvil, 1, "message_0058,!!downloaded!0!!,!!installed!0!!,!!verified!0!!,!!lines!0!!");
update_progress($anvil, 2, "message_0033");
# Make sure maintenance mode is enabled.
$anvil->System->maintenance_mode({set => 1}) if $anvil->data->{sys}{database}{connections};
# Run the update
run_os_update($anvil, 3);
# If we had no database, try to reconnect now tha
if (not $anvil->data->{sys}{database}{connections})
{
# Start the anvil-daemon, the caller likely called without a DB because we're being updated by
# striker-update-cluster, and so there will be a job waiting for us.
$anvil->System->enable_daemon({now => 1, daemon => "anvil-daemon"});
$anvil->System->enable_daemon({now => 1, daemon => "scancore"});
$anvil->Database->connect;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0132"});
if ($anvil->data->{sys}{database}{connections})
{
# If there's a job for us waiting, mark it as almost done.
my $query = "
SELECT
job_uuid
FROM
jobs
WHERE
job_command LIKE '%".$THIS_FILE."%'
AND
job_host_uuid = ".$anvil->Database->quote($anvil->Get->host_uuid)."
AND
job_progress = 0
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $job_uuid = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
if ($job_uuid)
{
$anvil->data->{jobs}{job_uuid} = $job_uuid;
$anvil->data->{switches}{'job-uuid'} = $job_uuid;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'jobs::job_uuid' => $anvil->data->{jobs}{job_uuid},
'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'},
}});
update_progress($anvil, 0, "clear");
update_progress($anvil, 90, "message_0324");
}
}
}
# We're done updating
my $reboot_needed = $anvil->System->reboot_needed({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { reboot_needed => $reboot_needed }});
if ($reboot_needed)
{
if (not $anvil->data->{switches}{'no-reboot'})
{
# Clear maintenance mode.
$anvil->System->maintenance_mode({set => 0}) if $anvil->data->{sys}{database}{connections};
# Record that we're rebooting so that 'striker-update-cluster' knows to wait for a reboot.
if ($anvil->data->{switches}{'job-uuid'})
{
my $query = "
UPDATE
jobs
SET
job_data = 'rebooted',
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
# Register a job to reboot.
update_progress($anvil, 98, "message_0318");
# Record that we're going to reboot now.
update_progress($anvil, 100, "message_0317");
if ($anvil->data->{sys}{database}{connections})
{
my ($job_uuid) = $anvil->Database->insert_or_update_jobs({
file => $THIS_FILE,
line => __LINE__,
job_command => $anvil->data->{path}{exe}{'anvil-manage-power'}." --reboot -y".$anvil->Log->switches,
job_data => "",
job_name => "reboot::system",
job_title => "job_0009",
job_description => "job_0006",
job_progress => 0,
job_status => "anvil_startup",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { job_uuid => $job_uuid }});
# If we got a job UUID, unlink the cache file.
if (-e $anvil->data->{path}{data}{reboot_cache})
{
unlink $anvil->data->{path}{data}{reboot_cache};
}
}
}
else
{
# Record that a reboot is needed.
update_progress($anvil, 100, "message_0039");
}
}
else
{
update_progress($anvil, 100, "message_0040");
}
# Clear maintenance mode.
$anvil->System->maintenance_mode({set => 0}) if $anvil->data->{sys}{database}{connections};
$anvil->nice_exit({exit_code => 0});
#############################################################################################################
# Functions #
#############################################################################################################
# This updates the progress if we were called with a job UUID.
sub update_progress
{
my ($anvil, $progress, $message) = @_;
# Log the progress percentage.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
progress => $progress,
message => $message,
"jobs::job_uuid" => $anvil->data->{jobs}{job_uuid},
}});
if ($progress > 100)
{
$progress = 100;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }});
}
if ($anvil->data->{jobs}{job_uuid})
{
$anvil->Job->update_progress({
debug => 2,
'print' => 1,
progress => $progress,
message => $message,
job_uuid => $anvil->data->{jobs}{job_uuid},
});
}
return(0);
}
# This updates the OS.
sub run_os_update
{
my ($anvil, $progress) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { progress => $progress }});
# This needs to be set to avoid warnings when called without a job-uuid.
$anvil->data->{sys}{last_update} = 0;
# Make sure that, if we're a node, we're out of the cluster.
my $host_type = $anvil->Get->host_type();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
if ($host_type eq "node")
{
# Call anvil-safe-stop
update_progress($anvil, $progress++, "message_0314");
my $problem = $anvil->Cluster->parse_cib();
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if (not $problem)
{
# Call anvil-safe-stop
update_progress($anvil, $progress++, "message_0315");
my $shell_call = $anvil->data->{path}{exe}{'anvil-safe-stop'};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# Something went wrong, abort.
update_progress($anvil, 100, "error_0420,!!return_code!".$return_code."!!,!!output!".$output."!!");
# Set the job_data to 'failed' so that striker-update-cluster' knows to abort.
if ($anvil->data->{switches}{'job-uuid'})
{
my $query = "
UPDATE
jobs
SET
job_data = 'failed',
modified_date = ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
WHERE
job_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'job-uuid'})."
;";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
$anvil->Database->write({query => $query, source => $THIS_FILE, line => __LINE__});
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } });
$anvil->nice_exit({exit_code => 4});
}
}
}
# Should we clear the cache?
if ($anvil->data->{switches}{'clear-cache'})
{
# Yes.
my $shell_call = $anvil->data->{path}{exe}{dnf}." clean all";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
update_progress($anvil, 5, "message_0316");
}
# Before we start, do we need to remove our locally build DRBD kernel modules?
my $package_changes = 0;
my $remove_drbd_kmod = 0;
my $shell_call = $anvil->data->{path}{exe}{dnf}." check-update";
open (my $file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
while(<$file_handle>)
{
chomp;
my $line = $anvil->Words->clean_spaces({string => $_});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if (($line =~ /kmod-drbd/) or ($line =~ /kernel/))
{
# Looks like it.
$remove_drbd_kmod = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { remove_drbd_kmod => $remove_drbd_kmod }});
last;
}
}
close $file_handle;
# So, shall we?
if ($remove_drbd_kmod)
{
# Yes, remove
update_progress($anvil, $progress++, "message_0320");
my $versions_to_remove = "";
my $shell_call = $anvil->data->{path}{exe}{dnf}." list installed";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
foreach my $line (split/\n/, $output)
{
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
if ($line =~ /(kmod-drbd-\d+.*?)\s/)
{
$versions_to_remove .= $1." ";
$package_changes++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
versions_to_remove => $versions_to_remove,
package_changes => $package_changes,
}});
}
}
# Now remove those packages.
update_progress($anvil, $progress++, "message_0321");
$shell_call = $anvil->data->{path}{exe}{dnf}." -y remove ".$versions_to_remove;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
open ($file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
while(<$file_handle>)
{
chomp;
my $line = $anvil->Words->clean_spaces({string => $_});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
}
close $file_handle;
}
# NOTE: We run this directly to better monitor progress and update the progress.
my $transaction_shown = 0;
my $success = 0;
my $to_update = 0;
my $percent_step = 0;
my $counted_lines = 0;
my $next_step = 0;
my $verifying = 0;
my $output = "";
$shell_call = $anvil->data->{path}{exe}{dnf}." -y update; ".$anvil->data->{path}{exe}{echo}." return_code:\$?";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
open ($file_handle, $shell_call." 2>&1 |") or $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, priority => "err", key => "log_0014", variables => { shell_call => $shell_call, error => $! }});
while(<$file_handle>)
{
chomp;
my $line = $_;
$output .= $line."\n";
$line = $anvil->Words->clean_spaces({string => $line});
$anvil->data->{counts}{lines}++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "counts::lines" => $anvil->data->{counts}{lines}, line => $line }});
# If there were no updates, let the user know.
if ($line =~ /^Nothing to do/i)
{
update_progress($anvil, 95, "message_0057");
}
if ((not $verifying) && ($line =~ /^Verifying /i))
{
# Update done, verifying now.
$verifying = 1;
update_progress($anvil, $progress++, "message_0038");
}
if ($line =~ /Running transaction/i)
{
# Done downloading
if (not $transaction_shown)
{
update_progress($anvil, $progress++, "message_0037");
$transaction_shown = 1;
}
}
if ($line =~ /return_code:(\d+)$/)
{
my $return_code = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { return_code => $return_code }});
if ($return_code == 0)
{
$success = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { success => $success }});
}
}
if (($line =~ / (\d+) Packages$/i) or ($line =~ / (\d+) Package$/i))
{
my $counted_lines = $1;
$package_changes = $counted_lines;
$to_update += $counted_lines;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
package_changes => $package_changes,
counted_lines => $counted_lines,
to_update => $to_update,
}});
}
if ($line =~ /Total download size: (.*)$/i)
{
my $update_size = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { update_size => $update_size }});
# Ready to install, update to 5%. The next step will count up to 95%.
update_progress($anvil, $progress++, "message_0035,!!size!$update_size!!");
# The total (reliable) count of events is (to_update * 3), counting '(x/y): '
# (download), 'Upgrading '/'Installing ' and 'Verifying '. We ignore the scriplet
# and other lines as it's hard to predict how many there will be, and they pass fast
# enough to not really matter for a progress bar.
$to_update *= 4;
$percent_step = $anvil->Convert->round({number => ($to_update / 90)});
$next_step = $percent_step;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
to_update => $to_update,
percent_step => $percent_step,
next_step => $next_step,
}});
}
# If 'percent_step' is set, we're ready to start counting lines.
if (($percent_step) && (($line =~ /\(\d+\/\d+\): /) or ($line =~ /^Upgrading /i) or ($line =~ /^Installing /) or ($line =~ /^Cleanup /i) or ($line =~ /^Verifying /i)))
{
$counted_lines++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { counted_lines => $counted_lines }});
if ($line =~ /\(\d+\/\d+\): /)
{
$anvil->data->{counts}{downloaded}++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "counts::downloaded" => $anvil->data->{counts}{downloaded} }});
}
if (($line =~ /^Upgrading /i) or ($line =~ /^Installing /))
{
$anvil->data->{counts}{installed}++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "counts::installed" => $anvil->data->{counts}{installed} }});
}
if ($line =~ /^Verifying /i)
{
$anvil->data->{counts}{verified}++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { "counts::verified" => $anvil->data->{counts}{verified} }});
}
if ($counted_lines > $next_step)
{
# Step up the progress.
$next_step += $percent_step;
$progress++;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
next_step => $next_step,
progress => $progress,
}});
$progress = 95 if $progress > 95;
update_progress($anvil, $progress, "");
}
}
# Update the progress if it's been more than a second since the last update.
if (time > $anvil->data->{sys}{last_update})
{
$progress = 95 if $progress > 95;
update_progress($anvil, $progress, "");
}
}
close $file_handle;
# Reload daemons to pick up any changed systemctl daemons.
$shell_call = $anvil->data->{path}{exe}{systemctl}." daemon-reload";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($systemctl_output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { systemctl_output => $systemctl_output, return_code => $return_code }});
### See if the kernel has been updated.
# Get the newest installed kernel
$shell_call = $anvil->data->{path}{exe}{rpm}." -q kernel | ".$anvil->data->{path}{exe}{'sort'}." | ".$anvil->data->{path}{exe}{tail}." -n 1";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
(my $installed_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
installed_kernel => $installed_kernel,
return_code => $return_code,
}});
$installed_kernel =~ s/^kernel-(\d+.\d+\.\d+-\d+)\..*$/$1/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { installed_kernel => $installed_kernel }});
# Get the running kernel
$shell_call = $anvil->data->{path}{exe}{uname}." -r";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
(my $active_kernel, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
active_kernel => $active_kernel,
return_code => $return_code,
}});
$active_kernel =~ s/(\d+.\d+\.\d+-\d+)\..*$/$1/;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { active_kernel => $active_kernel }});
if ($installed_kernel ne $active_kernel)
{
# Reboot needed
$anvil->data->{sys}{reboot} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::reboot' => $anvil->data->{sys}{reboot} }});
}
# If we installed and packages, and '--reboot' was given, reboot anyway.
if (($package_changes) && ($anvil->data->{switches}{reboot}))
{
# Reboot needed
$anvil->data->{sys}{reboot} = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::reboot' => $anvil->data->{sys}{reboot} }});
my $reboot_needed = $anvil->System->reboot_needed({set => 1});
}
# Did it work?
if (not $success)
{
# Nope.
update_progress($anvil, 0, "message_0036");
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, secure => 0, key => "error_0035", variables => { output => $output } });
sleep 5;
$anvil->nice_exit({exit_code => 3});
}
# Run anvil-version-changes.
$shell_call = $anvil->data->{path}{exe}{'anvil-version-changes'}.$anvil->Log->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
return(0);
};
Reference in New Issue View Git Blame Copy Permalink