* Changed Storage->read_file() to default 'force_read' to '1', causing it to always read from disk and ignore cache unless set to '0'.

* After much time wasted chasing a dnf bug (https://bugzilla.redhat.com/show_bug.cgi?id=1641947), tools/anvil-manage-install-target now populates the <DOCROOT>/<os_type>/<os_arch>/os/Packages/ directory with needed RPMs.

Signed-off-by: Digimer <digimer@alteeve.ca>

Anvil/Tools.pm

@@ -956,6 +956,7 @@ sub _set_paths
 				psql				=>	"/usr/bin/psql",
 				'postgresql-setup'		=>	"/usr/bin/postgresql-setup",
 				pwd				=>	"/usr/bin/pwd",
+				rpm				=>	"/usr/bin/rpm",
 				rsync				=>	"/usr/bin/rsync",
 				sed				=>	"/usr/bin/sed", 
 				'shutdown'			=>	"/usr/sbin/shutdown",

Anvil/Tools/Storage.pm

@@ -1185,9 +1185,9 @@ This is an optional parameter that controls whether the file is cached in case s
 
 This is the name of the file to read. When reading from a remote machine, it must be a full path and file name.
 
-=head3 force_read (optional)
+=head3 force_read (optional, default '1')
 
-This is an otpional parameter that, if set, forces the file to be read, bypassing cache if it exists. Set this to C<< 1 >> to bypass the cache.
+This is an otpional parameter that, if set to C<< 0 >>, will allow an existing cached copy of the file to be used instead of actually reading the file from disk (again).
 
 =head3 password (optional)
 
@@ -1224,7 +1224,7 @@ sub read_file
 	my $body        = "";
 	my $cache       = defined $parameter->{cache}       ? $parameter->{cache}       : 1;
 	my $file        = defined $parameter->{file}        ? $parameter->{file}        : "";
-	my $force_read  = defined $parameter->{force_read}  ? $parameter->{force_read}  : 0;
+	my $force_read  = defined $parameter->{force_read}  ? $parameter->{force_read}  : 1;
 	my $password    = defined $parameter->{password}    ? $parameter->{password}    : "";
 	my $port        = defined $parameter->{port}        ? $parameter->{port}        : 22;
 	my $remote_user = defined $parameter->{remote_user} ? $parameter->{remote_user} : "";

Anvil/Tools/Words.pm

@@ -207,6 +207,18 @@ sub key
 		}
 	}
 	
+	if ($string eq "#!not_found!#")
+	{
+		my $message = "[ Error ] - Failed to find the string key: [".$key."]!!";
+		print $THIS_FILE." ".__LINE__."; ".$message."\n";
+		$anvil->Log->entry({
+			level    => 0,
+			line     => __LINE__,
+			raw      => $message,
+			source   => $THIS_FILE,
+		})
+	}
+	
 	#print $THIS_FILE." ".__LINE__."; [ Debug ] - string: [$string]\n";
 	return($string);
 }

share/words.xml

@@ -115,9 +115,14 @@ The '-y' option prevents a confirmation prompt.
 		<key name="message_0075">The peer: [#!variable!host!#] is now in the database. Proceeding.</key>
 		<key name="message_0076">The job the peer add us has been registered. It should add us as soon as it looks for new jobs (generally within a second or two).</key>
 		<key name="message_0077">
-About to try to download packages needed for installing Anvil! machines to:
+About to try to download aproximately: [#!variable!packages!#] packages needed for installing Anvil! machines to:
 - [#!variable!directory!#].
-Please be patient! This could take a while to resolve dependencies and start downloading.
+
+NOTE: Please be patient! 
+      The 'dnf' cache will be cleared to ensure the freshest RPMs are download. This will cause a delay 
+      before output starts to appear. Once started, each RPM will be reported after it is downloaded. Large
+      RPMs may cause the output to appear stalled. You can verify that the download it proceeding by using
+      'df -s #!variable!directory!#' to verify the number is increasing.
 		</key>
 		<key name="message_0078">Output: [#!variable!line!#].</key>
 		<key name="message_0079">Error:  [#!variable!line!#].</key>

tools/anvil-manage-install-target

@@ -12,6 +12,7 @@
 # 
 # Exit codes;
 # 0 = Normal exit.
+# 1 = Alteeve repo not installed and not installable.
 # 
 # TODO:
 # - Support building the install target by mounting the ISO and checking /mnt/shared/incoming for needed 
@@ -45,6 +46,9 @@ $anvil->Get->switches;
 # load the list of packages;
 load_packages($anvil);
 
+# Make sure our repo is installed. If not, install it or fail out.
+#check_alteeve_repo($anvil);
+
 # Store the RPMs in repo directory.
 update_install_source($anvil);
 
@@ -56,6 +60,58 @@ $anvil->nice_exit({exit_code => 0});
 # Private functions.                                                                                        #
 #############################################################################################################
 
+sub check_alteeve_repo
+{
+	my ($anvil) = @_;
+	
+	my ($os_type, $os_arch) = $anvil->System->get_os_type({debug => 2});
+	my $version = ($os_type =~ /(\d+)$/)[0];
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		os_type => $os_type, 
+		os_arch => $os_arch, 
+		version => $version, 
+	}});
+	
+	my $repo_url  = "";
+	my $repo_file = "";
+	if ($os_type eq "fedora")
+	{
+		$repo_file = "/etc/yum.repos.d/alteeve-f".$version.".repo";
+		$repo_url  = "https://alteeve.com/an-repo/f".$version."/alteeve-f".$version."-repo-latest.noarch.rpm";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			repo_file => $repo_file,
+			repo_url  => $repo_url, 
+		}});
+	}
+	else
+	{
+		### TODO: Update this when rhel8 is out
+		$repo_file = "/etc/yum.repos.d/alteeve-el".$version.".repo";
+		$repo_url  = "https://alteeve.com/an-repo/el".$version."/alteeve-el".$version."-repo-latest.noarch.rpm";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			repo_file => $repo_file,
+			repo_url  => $repo_url, 
+		}});
+	}
+	
+	# If the repo file doesn't exist, try to install it.
+	if (not -e $repo_file)
+	{
+		# Install the repo
+		my $handle = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{rpm}." -Uvh ".$repo_url });
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { handle => $handle }});
+	}
+	
+	# If it still doesn't exist, we're done.
+	if (not -e $repo_file)
+	{
+		# Die
+		die "Failed to install the Alteeve repo, unable to proceed.\n";
+	}
+	
+	return(0);
+}
+
 # This uses yumdownloader to push the files into '/var/www/html/<os_name>/<arch>/os
 sub update_install_source
 {
@@ -69,77 +125,79 @@ sub update_install_source
 	
 	### TODO: Make sure this path exists.
 	my $download_path = "/var/www/html/".$os_type."/".$os_arch."/os/Packages/";
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { download_path => $download_path }});
+	my $array_size    = @{$anvil->data->{packages}};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		download_path => $download_path, 
+		array_size    => $array_size,
+	}});
 	if (not -e $download_path)
 	{
 		$anvil->Storage->make_directory({debug => 2, directory => $download_path, mode => "0775"});
 	}
-	print $anvil->Words->string({key => "message_0077", variables => { directory => $download_path }})."\n";
+	print $anvil->Words->string({key => "message_0077", variables => { 
+		directory => $download_path, 
+		packages  => $anvil->Convert->add_commas({number => $array_size}), 
+	}})."\n";
 	
-	# Assemble the download command.
+	# Clear the dnf cache
-	my $shell_call = $anvil->data->{path}{exe}{dnf}." download --destdir ".$download_path." --resolve";
+	my $handle = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{dnf}." clean expire-cache" });
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { handle => $handle }});
+	
+	my $shell_call = $anvil->data->{path}{exe}{dnf}." download --destdir ".$download_path." ";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 	foreach my $package (sort {$a cmp $b} @{$anvil->data->{packages}})
 	{
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'package' => $package }});
+		# Append the package to the active shell call.
 		$shell_call .= " ".$package;
 	}
 	
 	# Now call the command in the background, then we'll track the output.
 	my $stdout_file = "/tmp/".$THIS_FILE."_update_install_source.stdout";
-	my $stderr_file = "/tmp/".$THIS_FILE."_update_install_source.stderr";
+	if (-e $stdout_file)
+	{
+		unlink $stdout_file;
+	}
 	my $handle = $anvil->System->call({
 		debug       => 2,
 		shell_call  => $shell_call, 
 		background  => 1, 
 		stdout_file => $stdout_file, 
-		stderr_file => $stderr_file, 
 	});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { handle => $handle }});
 	
 	# Now we'll loop, printing output, until the handle dies.
-	my $alive       = 1;
+	my $alive            = 1;
-	my $stdout_line = 0;
+	my $last_stdout_line = 0;
-	my $stderr_line = 0;
 	while($alive)
 	{
-		# read the stdout and stderr files and print any lines we've not seen yet.
+		# Are we still alive?
-		my $stdout = $anvil->Storage->read_file({file => $stdout_file});
+		$alive = $handle->poll();
-		my $stderr = $anvil->Storage->read_file({file => $stderr_file});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alive => $alive }});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		
-			stdout => $stdout,
+		# Sleep (even if we're dead now, we want to give a second for the stdout file to be updated 
-			stderr => $stderr, 
+		# before processing it).
-		}});
+		sleep 1;
+		
+		# print any new STDOUT lines
+		my $stdout = $anvil->Storage->read_file({force_read => 1, file => $stdout_file});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { stdout => $stdout }});
+		
 		my $this_stdout_line = 0;
-		my $this_stderr_line = 0;
 		foreach my $line (split/\n/, $stdout)
 		{
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
 			$this_stdout_line++;
-			if ($this_stdout_line > $stdout_line)
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { 
+				's1:this_stdout_line' => $this_stdout_line, 
+				's2:last_stdout_line' => $last_stdout_line, 
+				's3:line'             => $line,
+			}});
+			if ($this_stdout_line > $last_stdout_line)
 			{
-				$this_stdout_line = $stdout_line;
 				print $anvil->Words->string({key => "message_0078", variables => { line => $line }})."\n";
-			}
-		}
-		foreach my $line (split/\n/, $stderr)
-		{
-			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }});
-			$this_stderr_line++;
-			if ($this_stderr_line > $stderr_line)
-			{
-				$this_stderr_line = $stderr_line;
-				print $anvil->Words->string({key => "message_0079", variables => { line => $line }})."\n";
-			}
-		}
 				
-		# Are we still alive?
+				$last_stdout_line = $this_stdout_line;
-		$alive = $handle->poll();
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { last_stdout_line => $last_stdout_line }});
-		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { alive => $alive }});
+			}
-		
-		if ($alive)
-		{
-			sleep 1;
 		}
 	}
 	
@@ -203,7 +261,6 @@ sub load_packages
 		"alsa-plugins-pulseaudio.x86_64",
 		"alsa-ucm.x86_64",
 		"alsa-utils.x86_64",
-		"alteeve-f28-repo.noarch",
 		"amtterm.x86_64",
 		"anvil-core.noarch",
 		"anvil-dr.noarch",