From 706052105e68038f491fe0ce06fcd3b79b6755e1 Mon Sep 17 00:00:00 2001 From: Digimer Date: Wed, 24 Oct 2018 02:21:41 -0400 Subject: [PATCH] * Changed Storage->read_file() to default 'force_read' to '1', causing it to always read from disk and ignore cache unless set to '0'. * After much time wasted chasing a dnf bug (https://bugzilla.redhat.com/show_bug.cgi?id=1641947), tools/anvil-manage-install-target now populates the ///os/Packages/ directory with needed RPMs. Signed-off-by: Digimer --- Anvil/Tools.pm | 1 + Anvil/Tools/Storage.pm | 6 +- Anvil/Tools/Words.pm | 12 +++ share/words.xml | 9 +- tools/anvil-manage-install-target | 139 +++++++++++++++++++++--------- 5 files changed, 121 insertions(+), 46 deletions(-) diff --git a/Anvil/Tools.pm b/Anvil/Tools.pm index c5f25026..6a3854c7 100755 --- a/Anvil/Tools.pm +++ b/Anvil/Tools.pm @@ -956,6 +956,7 @@ sub _set_paths psql => "/usr/bin/psql", 'postgresql-setup' => "/usr/bin/postgresql-setup", pwd => "/usr/bin/pwd", + rpm => "/usr/bin/rpm", rsync => "/usr/bin/rsync", sed => "/usr/bin/sed", 'shutdown' => "/usr/sbin/shutdown", diff --git a/Anvil/Tools/Storage.pm b/Anvil/Tools/Storage.pm index 7b9e2364..acafcf6f 100755 --- a/Anvil/Tools/Storage.pm +++ b/Anvil/Tools/Storage.pm @@ -1185,9 +1185,9 @@ This is an optional parameter that controls whether the file is cached in case s This is the name of the file to read. When reading from a remote machine, it must be a full path and file name. -=head3 force_read (optional) +=head3 force_read (optional, default '1') -This is an otpional parameter that, if set, forces the file to be read, bypassing cache if it exists. Set this to C<< 1 >> to bypass the cache. +This is an otpional parameter that, if set to C<< 0 >>, will allow an existing cached copy of the file to be used instead of actually reading the file from disk (again). =head3 password (optional) @@ -1224,7 +1224,7 @@ sub read_file my $body = ""; my $cache = defined $parameter->{cache} ? $parameter->{cache} : 1; my $file = defined $parameter->{file} ? $parameter->{file} : ""; - my $force_read = defined $parameter->{force_read} ? $parameter->{force_read} : 0; + my $force_read = defined $parameter->{force_read} ? $parameter->{force_read} : 1; my $password = defined $parameter->{password} ? $parameter->{password} : ""; my $port = defined $parameter->{port} ? $parameter->{port} : 22; my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : ""; diff --git a/Anvil/Tools/Words.pm b/Anvil/Tools/Words.pm index 3939cdb3..49a7c785 100755 --- a/Anvil/Tools/Words.pm +++ b/Anvil/Tools/Words.pm @@ -207,6 +207,18 @@ sub key } } + if ($string eq "#!not_found!#") + { + my $message = "[ Error ] - Failed to find the string key: [".$key."]!!"; + print $THIS_FILE." ".__LINE__."; ".$message."\n"; + $anvil->Log->entry({ + level => 0, + line => __LINE__, + raw => $message, + source => $THIS_FILE, + }) + } + #print $THIS_FILE." ".__LINE__."; [ Debug ] - string: [$string]\n"; return($string); } diff --git a/share/words.xml b/share/words.xml index d4f8e512..45d44056 100644 --- a/share/words.xml +++ b/share/words.xml @@ -115,9 +115,14 @@ The '-y' option prevents a confirmation prompt. The peer: [#!variable!host!#] is now in the database. Proceeding. The job the peer add us has been registered. It should add us as soon as it looks for new jobs (generally within a second or two). -About to try to download packages needed for installing Anvil! machines to: +About to try to download aproximately: [#!variable!packages!#] packages needed for installing Anvil! machines to: - [#!variable!directory!#]. -Please be patient! This could take a while to resolve dependencies and start downloading. + +NOTE: Please be patient! + The 'dnf' cache will be cleared to ensure the freshest RPMs are download. This will cause a delay + before output starts to appear. Once started, each RPM will be reported after it is downloaded. Large + RPMs may cause the output to appear stalled. You can verify that the download it proceeding by using + 'df -s #!variable!directory!#' to verify the number is increasing. Output: [#!variable!line!#]. Error: [#!variable!line!#]. diff --git a/tools/anvil-manage-install-target b/tools/anvil-manage-install-target index 4473bd26..76aa170b 100755 --- a/tools/anvil-manage-install-target +++ b/tools/anvil-manage-install-target @@ -12,6 +12,7 @@ # # Exit codes; # 0 = Normal exit. +# 1 = Alteeve repo not installed and not installable. # # TODO: # - Support building the install target by mounting the ISO and checking /mnt/shared/incoming for needed @@ -45,6 +46,9 @@ $anvil->Get->switches; # load the list of packages; load_packages($anvil); +# Make sure our repo is installed. If not, install it or fail out. +#check_alteeve_repo($anvil); + # Store the RPMs in repo directory. update_install_source($anvil); @@ -56,6 +60,58 @@ $anvil->nice_exit({exit_code => 0}); # Private functions. # ############################################################################################################# +sub check_alteeve_repo +{ + my ($anvil) = @_; + + my ($os_type, $os_arch) = $anvil->System->get_os_type({debug => 2}); + my $version = ($os_type =~ /(\d+)$/)[0]; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + os_type => $os_type, + os_arch => $os_arch, + version => $version, + }}); + + my $repo_url = ""; + my $repo_file = ""; + if ($os_type eq "fedora") + { + $repo_file = "/etc/yum.repos.d/alteeve-f".$version.".repo"; + $repo_url = "https://alteeve.com/an-repo/f".$version."/alteeve-f".$version."-repo-latest.noarch.rpm"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + repo_file => $repo_file, + repo_url => $repo_url, + }}); + } + else + { + ### TODO: Update this when rhel8 is out + $repo_file = "/etc/yum.repos.d/alteeve-el".$version.".repo"; + $repo_url = "https://alteeve.com/an-repo/el".$version."/alteeve-el".$version."-repo-latest.noarch.rpm"; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + repo_file => $repo_file, + repo_url => $repo_url, + }}); + } + + # If the repo file doesn't exist, try to install it. + if (not -e $repo_file) + { + # Install the repo + my $handle = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{rpm}." -Uvh ".$repo_url }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { handle => $handle }}); + } + + # If it still doesn't exist, we're done. + if (not -e $repo_file) + { + # Die + die "Failed to install the Alteeve repo, unable to proceed.\n"; + } + + return(0); +} + # This uses yumdownloader to push the files into '/var/www/html///os sub update_install_source { @@ -69,78 +125,80 @@ sub update_install_source ### TODO: Make sure this path exists. my $download_path = "/var/www/html/".$os_type."/".$os_arch."/os/Packages/"; - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { download_path => $download_path }}); + my $array_size = @{$anvil->data->{packages}}; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { + download_path => $download_path, + array_size => $array_size, + }}); if (not -e $download_path) { $anvil->Storage->make_directory({debug => 2, directory => $download_path, mode => "0775"}); } - print $anvil->Words->string({key => "message_0077", variables => { directory => $download_path }})."\n"; + print $anvil->Words->string({key => "message_0077", variables => { + directory => $download_path, + packages => $anvil->Convert->add_commas({number => $array_size}), + }})."\n"; - # Assemble the download command. - my $shell_call = $anvil->data->{path}{exe}{dnf}." download --destdir ".$download_path." --resolve"; + # Clear the dnf cache + my $handle = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{dnf}." clean expire-cache" }); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { handle => $handle }}); + + my $shell_call = $anvil->data->{path}{exe}{dnf}." download --destdir ".$download_path." "; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); foreach my $package (sort {$a cmp $b} @{$anvil->data->{packages}}) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'package' => $package }}); + # Append the package to the active shell call. $shell_call .= " ".$package; } # Now call the command in the background, then we'll track the output. my $stdout_file = "/tmp/".$THIS_FILE."_update_install_source.stdout"; - my $stderr_file = "/tmp/".$THIS_FILE."_update_install_source.stderr"; + if (-e $stdout_file) + { + unlink $stdout_file; + } my $handle = $anvil->System->call({ debug => 2, shell_call => $shell_call, background => 1, stdout_file => $stdout_file, - stderr_file => $stderr_file, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { handle => $handle }}); # Now we'll loop, printing output, until the handle dies. - my $alive = 1; - my $stdout_line = 0; - my $stderr_line = 0; + my $alive = 1; + my $last_stdout_line = 0; while($alive) { - # read the stdout and stderr files and print any lines we've not seen yet. - my $stdout = $anvil->Storage->read_file({file => $stdout_file}); - my $stderr = $anvil->Storage->read_file({file => $stderr_file}); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { - stdout => $stdout, - stderr => $stderr, - }}); + # Are we still alive? + $alive = $handle->poll(); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alive => $alive }}); + + # Sleep (even if we're dead now, we want to give a second for the stdout file to be updated + # before processing it). + sleep 1; + + # print any new STDOUT lines + my $stdout = $anvil->Storage->read_file({force_read => 1, file => $stdout_file}); + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { stdout => $stdout }}); + my $this_stdout_line = 0; - my $this_stderr_line = 0; foreach my $line (split/\n/, $stdout) { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); $this_stdout_line++; - if ($this_stdout_line > $stdout_line) + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { + 's1:this_stdout_line' => $this_stdout_line, + 's2:last_stdout_line' => $last_stdout_line, + 's3:line' => $line, + }}); + if ($this_stdout_line > $last_stdout_line) { - $this_stdout_line = $stdout_line; print $anvil->Words->string({key => "message_0078", variables => { line => $line }})."\n"; + + $last_stdout_line = $this_stdout_line; + $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { last_stdout_line => $last_stdout_line }}); } } - foreach my $line (split/\n/, $stderr) - { - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); - $this_stderr_line++; - if ($this_stderr_line > $stderr_line) - { - $this_stderr_line = $stderr_line; - print $anvil->Words->string({key => "message_0079", variables => { line => $line }})."\n"; - } - } - - # Are we still alive? - $alive = $handle->poll(); - $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alive => $alive }}); - - if ($alive) - { - sleep 1; - } } return(0); @@ -203,7 +261,6 @@ sub load_packages "alsa-plugins-pulseaudio.x86_64", "alsa-ucm.x86_64", "alsa-utils.x86_64", - "alteeve-f28-repo.noarch", "amtterm.x86_64", "anvil-core.noarch", "anvil-dr.noarch",