Merge pull request #309 from ClusterLabs/anvil-tools-dev

* Fixed a bug in Network->load_ips() where interfaces were being mark…

Anvil/Tools.pm

@@ -1102,6 +1102,7 @@ sub _set_paths
 				html				=>	"/var/www/html",
 				ifcfg				=>	"/etc/sysconfig/network-scripts",
 				journald			=>	"/var/log/journal", 
+				libvirtd_definitions		=>	"/etc/libvirt/qemu/",
 				pgsql				=>	"/var/lib/pgsql/",
 				resource_status			=>	"/sys/kernel/debug/drbd/resources",
 				scan_agents			=>	"/usr/sbin/scancore-agents",

Anvil/Tools/Database.pm

@@ -3244,16 +3244,62 @@ FROM
 			"file_locations::file_location_uuid::${file_location_uuid}::modified_date"           => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{modified_date}, 
 		}});
 		
-		### TODO: Remove this when the WebUI is updated.
+		my $file_name      = "";
+		my $file_directory = "";
+		my $file_size      = "";
+		my $file_md5sum    = "";
+		my $file_type      = "";
+		my $file_mtime     = "";
+		if (exists $anvil->data->{files}{file_uuid}{$file_location_file_uuid})
+		{
+			$file_name      = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_name};
+			$file_directory = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_directory};
+			$file_size      = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_size}; 
+			$file_md5sum    = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_md5sum};
+			$file_type      = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_type};
+			$file_mtime     = $anvil->data->{files}{file_uuid}{$file_location_file_uuid}{file_mtime};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+				file_name      => $file_name, 
+				file_directory => $file_directory, 
+				file_size      => $file_size." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $file_size}).")", 
+				file_md5sum    => $file_md5sum, 
+				file_type      => $file_type, 
+				file_mtime     => $file_mtime,
+			}});
+		}
+		
 		# If this host is a node in an Anvil!, set the old 'file_location_anvil_uuid' to maintain 
 		# backwards compatibility.
 		if ((exists $anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}) && 
 		    ($anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}{anvil_uuid}))
 		{
-			$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_anvil_uuid} = $anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}{anvil_uuid};
+			my $anvil_uuid = $anvil->data->{hosts}{host_uuid}{$file_location_host_uuid}{anvil_uuid};
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { anvil_uuid => $anvil_uuid }});
+			
+			$anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_anvil_uuid} = $anvil_uuid;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
 				"file_locations::file_location_uuid::${file_location_uuid}::file_location_anvil_uuid" => $anvil->data->{file_locations}{file_location_uuid}{$file_location_uuid}{file_location_anvil_uuid}, 
 			}});
+			
+			if ($file_name)
+			{
+				$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}                    = $file_location_file_uuid;
+				$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name}      = $file_name;
+				$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory} = $file_directory;
+				$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}      = $file_size;
+				$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum}    = $file_md5sum;
+				$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type}      = $file_type;
+				$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime}     = $file_mtime;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
+					"anvils::anvil_uuid::${anvil_uuid}::file_name::${file_name}::file_uuid"                    => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}{$file_name}{file_uuid}, 
+					"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_name"      => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_name}, 
+					"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_directory" => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_directory}, 
+					"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_size"      => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_size}}).")", 
+					"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_md5sum"    => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_md5sum}, 
+					"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_type"      => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_type}, 
+					"anvils::anvil_uuid::${anvil_uuid}::file_uuid::${file_location_file_uuid}::file_mtime"     => $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_uuid}{$file_location_file_uuid}{file_mtime}, 
+				}});
+			}
 		}
 		
 		# Make it easy to find files by anvil and file UUID.

Anvil/Tools/Get.pm

@@ -802,6 +802,10 @@ ORDER BY
 		}});
 	}
 	
+	# This re-connects files available on each Anvil! node.
+	$anvil->Database->get_files({debug => 2});
+	$anvil->Database->get_file_locations({debug => 2});
+	
 	return(0);
 }
 

Anvil/Tools/Network.pm

@@ -2996,6 +2996,7 @@ AND
 			ip_address_on_uuid         => $ip_address_on_uuid,
 		}});
 		
+		my $device_type_with_ip    = $ip_address_on_type;
 		my $bridge_name            = "";
 		my $bond_name              = "";
 		my $interface_name         = "";
@@ -3031,7 +3032,7 @@ AND
 			   $bond_name        = $results->[0]->[0];
 			my $active_interface = $results->[0]->[1];
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { 
-				interface_name   => $interface_name, 
+				bond_name        => $bond_name, 
 				active_interface => $active_interface,
 			}});
 			
@@ -3045,6 +3046,18 @@ AND
 			}
 		}
 		
+		my $device_name_with_ip = "";
+		if ($ip_address_on_type eq "bridge")
+		{
+			$device_name_with_ip = $bridge_name;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { device_name_with_ip => $device_name_with_ip }});
+		}
+		elsif ($ip_address_on_type eq "bond")
+		{
+			$device_name_with_ip = $bond_name;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { device_name_with_ip => $device_name_with_ip }});
+		}
+		
 		if ($network_interface_uuid)
 		{
 			my $query = "
@@ -3083,18 +3096,6 @@ AND
 				interface_mac  => $interface_mac, 
 			}});
 			
-			# If this is a bridge or a bond, use that name for the interface.
-			if ($bridge_name)
-			{
-				$interface_name = $bridge_name;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
-			}
-			if ($bond_name)
-			{
-				$interface_name = $bond_name;
-				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { interface_name => $interface_name }});
-			}
-			
 			$anvil->data->{network}{$host}{interface}{$interface_name}{network_interface_uuid} =         $results->[0]->[0];
 			$anvil->data->{network}{$host}{interface}{$interface_name}{mac_address}            =         $interface_mac;
 			$anvil->data->{network}{$host}{interface}{$interface_name}{ip}                     =         $ip_address_address;
@@ -3102,7 +3103,7 @@ AND
 			$anvil->data->{network}{$host}{interface}{$interface_name}{default_gateway}        =         $ip_address_default_gateway;
 			$anvil->data->{network}{$host}{interface}{$interface_name}{gateway}                =         $ip_address_gateway;
 			$anvil->data->{network}{$host}{interface}{$interface_name}{dns}                    =         $ip_address_dns;
-			$anvil->data->{network}{$host}{interface}{$interface_name}{type}                   =         $ip_address_on_type;
+			$anvil->data->{network}{$host}{interface}{$interface_name}{type}                   =         "interface";
 			$anvil->data->{network}{$host}{interface}{$interface_name}{speed}                  =         $results->[0]->[3];
 			$anvil->data->{network}{$host}{interface}{$interface_name}{mtu}                    =         $results->[0]->[4];
 			$anvil->data->{network}{$host}{interface}{$interface_name}{link_state}             =         $results->[0]->[5];
@@ -3129,6 +3130,12 @@ AND
 				"network::${host}::interface::${interface_name}::bond_uuid"              => $anvil->data->{network}{$host}{interface}{$interface_name}{bond_uuid}, 
 				"network::${host}::interface::${interface_name}::bridge_uuid"            => $anvil->data->{network}{$host}{interface}{$interface_name}{bridge_uuid}, 
 			}});
+			
+			if (not $device_name_with_ip)
+			{
+				$device_name_with_ip = $interface_name;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { device_name_with_ip => $device_name_with_ip }});
+			}
 		}
 		elsif ($ip_address_on_type eq "bond")
 		{
@@ -4195,7 +4202,7 @@ sub _get_server_ports
 	my $debug     = defined $parameter->{debug} ? $parameter->{debug} : 3;
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0125", variables => { method => "Network->_get_server_ports()" }});
 	
-	my $shell_call = $anvil->data->{path}{exe}{virsh}." list --name";
+	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --name";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
 	
 	my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});
@@ -4216,7 +4223,7 @@ sub _get_server_ports
 	
 	foreach my $server (sort {$a cmp $b} @{$servers})
 	{
-		my $shell_call = $anvil->data->{path}{exe}{virsh}." dumpxml ".$server;
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml ".$server;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { shell_call => $shell_call }});
 		
 		my ($output, $return_code) = $anvil->System->call({debug => $debug, shell_call => $shell_call});

Anvil/Tools/Server.pm

@@ -613,6 +613,7 @@ sub get_runtime
 	return($runtime);
 }
 
+
 =head2 get_status
 
 This reads in a server's XML definition file from disk, if available, and from memory, if the server is running. The XML is analyzed and data is stored under C<< server::<target>::<server_name>::from_disk::x >> for data from the on-disk XML and C<< server::<target>>::<server_name>::from_virsh::x >>. 

Anvil/Tools/System.pm

@@ -241,6 +241,8 @@ sub call
 		redirect_stderr => $redirect_stderr, 
 		stderr_file     => $stderr_file, 
 		stdout_file     => $stdout_file, 
+		source          => $source,
+		line            => $line, 
 	}});
 	
 	my $return_code = 9999;
@@ -248,6 +250,13 @@ sub call
 	if (not $shell_call)
 	{
 		# wat?
+		$anvil->Log->variables({source => $source, line => $line, level => 1, secure => $secure, list => { 
+			background      => $background, 
+			shell_call      => $shell_call,
+			redirect        => $redirect, 
+			stderr_file     => $stderr_file, 
+			stdout_file     => $stdout_file, 
+		}});
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => "err", key => "log_0043"});
 	}
 	else
@@ -2725,7 +2734,7 @@ sub generate_state_json
 	$anvil->Network->load_ips({
 		debug     => $debug,
 		host      => $anvil->Get->short_host_name(),
-		host_uuid => $anvil->data->{sys}{host_uuid},
+		host_uuid => $anvil->Get->host_uuid,
 	});
 	
 	$anvil->data->{json}{all_systems}{hosts} = [];

notes

@@ -5,6 +5,10 @@ Common queries;
 * SELECT a.dr_link_uuid, b.host_name, c.anvil_name, a.dr_link_note FROM dr_links a, hosts b, anvils c WHERE a.dr_link_host_uuid = b.host_uuid AND a.dr_link_anvil_uuid = c.anvil_uuid ORDER BY c.anvil_name ASC, b.host_name ASC;
 
 
+uname -r; grubby --default-kernel; lsinitrd -m /boot/initramfs-4.18.0-448.el8.x86_64.img | grep lvm; systemctl is-enabled scancore.service; 
+dnf -y update; systemctl disable --now anvil-daemon; systemctl disable --now scancore
+
+
 When pairing Striker, make sure new config goes to all known nodes!
 
 

ocf/alteeve/server

@@ -1212,7 +1212,7 @@ sub server_status
 		{
 			   $loop       = 0;
 			my $found      = 0;
-			my $shell_call = $anvil->data->{path}{exe}{virsh}." list --all";
+			my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 			
 			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});

scancore-agents/scan-network/scan-network

@@ -488,7 +488,7 @@ sub collect_data
 		my $default_network = "/etc/libvirt/qemu/networks/default.xml";
 		if (-e $default_network)
 		{
-			my $shell_call = $anvil->data->{path}{exe}{virsh}." net-destroy default";
+			my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy default";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
@@ -496,7 +496,7 @@ sub collect_data
 				return_code => $return_code, 
 			}});
 			
-			$shell_call = $anvil->data->{path}{exe}{virsh}." net-undefine default";
+			$shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine default";
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 			($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
@@ -517,7 +517,7 @@ sub collect_data
 			});
 		}
 =cut
-		my $shell_call = $anvil->data->{path}{exe}{virsh}." net-list --all --name";
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-list --all --name";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 		
 		# This often hangs.
@@ -539,7 +539,7 @@ sub collect_data
 				
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "striker_0287", variables => { bridge => $line }});
 				
-				my $shell_call = $anvil->data->{path}{exe}{virsh}." net-destroy ".$line;
+				my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy ".$line;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
@@ -547,7 +547,7 @@ sub collect_data
 					return_code => $return_code, 
 				}});
 				
-				$shell_call = $anvil->data->{path}{exe}{virsh}." net-undefine ".$line;
+				$shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine ".$line;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 				($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 

scancore-agents/scan-server/scan-server

@@ -202,7 +202,7 @@ sub check_vnc
 			next if $server_state eq "crashed";
 			
 			# Get the VNC port. Ignore the IP and the port number is +5900. 
-			my $shell_call = $anvil->data->{path}{exe}{virsh}." vncdisplay --domain ".$server_name;
+			my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." vncdisplay --domain ".$server_name;
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 			
 			my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
@@ -359,7 +359,7 @@ sub collect_data
 		}
 	}
 	
-	my $shell_call = $anvil->data->{path}{exe}{virsh}." list --all";
+	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all";
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 	
 	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
@@ -531,7 +531,7 @@ DELETED   - Marks a server as no longer existing
 					# The definition was likely changed by the user while it was running. We 
 					# should have already picked up the changes, but just to be safe, check for
 					# differences.
-					my $shell_call = $anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
+					my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 					
 					my ($virsh_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
@@ -545,7 +545,7 @@ DELETED   - Marks a server as no longer existing
 					update_definitions_from_virsh($anvil, $server_name, $server_uuid, $virsh_definition);
 					
 					# Now undefine the server
-					$shell_call = $anvil->data->{path}{exe}{virsh}." undefine ".$server_name;
+					$shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server_name;
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 					
 					(my $output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
@@ -688,7 +688,7 @@ DELETED   - Marks a server as no longer existing
 			$anvil->Alert->register({alert_level => "notice", message => "scan_server_alert_0003", variables => $variables, set_by => $THIS_FILE});
 		}
 		
-		my $shell_call = $anvil->data->{path}{exe}{virsh}." dumpxml ".$server_name;
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml ".$server_name;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 		
 		my ($virsh_live_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
@@ -1018,7 +1018,7 @@ DELETED   - Marks a server as no longer existing
 	if ($peer_access)
 	{
 		# Get a list of servers running on our peer. 
-		my $shell_call = $anvil->data->{path}{exe}{virsh}." list --all";
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 		
 		my ($output, $error, $return_code) = $anvil->Remote->call({
@@ -1151,7 +1151,7 @@ sub get_and_parse_virsh_definition
 	my ($anvil, $server_name) = @_;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_name => $server_name }});
 
-	my $shell_call = $anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
+	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 	
 	my ($virsh_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
@@ -1180,7 +1180,7 @@ sub redefine_server_from_disk
 	
 	# Push the new definition into virsh (it won't take effect until a reboot likely, but it will update
 	# the 'inactive' definition immediately.
-	my $shell_call = $anvil->data->{path}{exe}{virsh}." defined ".$xml_file;
+	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." define ".$xml_file;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 	
 	my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
@@ -1190,7 +1190,7 @@ sub redefine_server_from_disk
 	}});
 	
 	# Now undefine the server again so it disappears when stopped. 
-	$shell_call = $anvil->data->{path}{exe}{virsh}." undefine ".$server_name;
+	$shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server_name;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 	
 	($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});

share/words.xml

@@ -1523,13 +1523,13 @@ Note: This is a permanent action! If you protect this server again later, a full
 		<key name="job_0458"><![CDATA[Available options
  --name          - Alphanumeric, 1~16 characters long.
  --os            - Run: 'osinfo-query os' and reference the 'Short ID' column for valid options.
- --cpu           - 1 ~ #!varaible!threads!#
+ --cpu           - 1 ~ #!variable!threads!#
- --ram           - Bytes or human readable, min is 64KiB, max is #!variable!max_ram.
+ --ram           - Bytes or human readable, min is 64KiB, max is #!variable!max_ram!#.
  --storage-group - Name or UUID. Valid options are:]]></key>
-		<key name="job_0459"><![CDATA[                     - Name: [#!variable!name!#], UUID: [#!variable!uuid!#], free space: [#!variable!free_space!#]]]></key>
+		<key name="job_0459"><![CDATA[                 > Name: [#!variable!name!#], UUID: [#!variable!uuid!#], free space: [#!variable!free_space!#]]]></key>
 		<key name="job_0460"><![CDATA[ --storage-size  - Disk size, see above for space available
  --install-media - File name or file UUID. Available discs are:]]></key>
-		<key name="job_0461"><![CDATA[                     - File name: [#!variable!name!#], file UUID: [#!variable!uuid!#], size: [#!variable!size!#]]]></key>
+		<key name="job_0461"><![CDATA[                 > File name: [#!variable!name!#], file UUID: [#!variable!uuid!#], size: [#!variable!size!#]]]></key>
 		<key name="job_0462"><![CDATA[ --driver-disc   - (optional) A driver disc to be added as a second optical drive. Valid options are above.]]></key>
 
 		<!-- Log entries -->

tools/anvil-configure-host

@@ -1412,7 +1412,7 @@ ORDER BY
 	
 	# If any virtio bridges exist, remove it/them.
 	my $start                   = 0;
-	my ($bridges, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{virsh}." net-list"});
+	my ($bridges, $return_code) = $anvil->System->call({debug => 2, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-list"});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { bridges => $bridges, return_code => $return_code }});
 	if ($return_code)
 	{
@@ -1475,15 +1475,15 @@ ORDER BY
 		foreach my $bridge (sort {$a cmp $b} keys %{$anvil->data->{virsh}{bridge}})
 		{
 			# Destroy (stop) it.
-			my ($destroy, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." net-destroy ".$bridge});
+			my ($destroy, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy ".$bridge});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { destroy => $destroy, return_code => $return_code }});
 			
 			# Disable it from auto-start.
-			(my $disable, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." net-autostart ".$bridge." --disable"});
+			(my $disable, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-autostart ".$bridge." --disable"});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { disable => $disable, return_code => $return_code }});
 			
 			# Undefine (delete)
-			(my $undefine, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." net-undefine ".$bridge});
+			(my $undefine, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine ".$bridge});
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { undefine => $undefine, return_code => $return_code }});
 		}
 	}

tools/anvil-get-server-screenshot

@@ -82,7 +82,7 @@ sub get_server_screenshot
 
 	my $resize_args = defined $parameters->{resize_args} ? $parameters->{resize_args} : '';
 
-	my $sh_domstate = $anvil->data->{path}{exe}{virsh}." domstate --domain ".$server_uuid;
+	my $sh_domstate = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." domstate --domain ".$server_uuid;
 	my ($sh_domstate_output, $sh_domstate_return_code) = system_call({ debug => $debug, shell_call => $sh_domstate });
 
 	if ( ($sh_domstate_return_code != 0) || ($sh_domstate_output ne "running") )
@@ -90,7 +90,7 @@ sub get_server_screenshot
 		return "";
 	}
 
-	my $shell_call = $anvil->data->{path}{exe}{virsh}." --quiet screenshot --domain ".$server_uuid." --file ".$anvil->data->{path}{devices}{stdout};
+	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." --quiet screenshot --domain ".$server_uuid." --file ".$anvil->data->{path}{devices}{stdout};
 
 	if ($resize_args =~ /^\d+x\d+$/)
 	{

tools/anvil-join-anvil

@@ -2056,8 +2056,8 @@ sub check_local_network
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0085"});
 		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0085");
 		
-		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{virsh}." net-destroy default"});
+		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy default"});
-		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{virsh}." net-undefine default "});
+		$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine default "});
 		
 		$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "job_0034"});
 		update_progress($anvil, ($anvil->data->{job}{progress} += 2), "job_0034");

tools/anvil-manage-firewall

@@ -132,7 +132,7 @@ sub wait_for_server
 	}});
 	while($waiting)
 	{
-		my $shell_call = $anvil->data->{path}{exe}{virsh}." list --name";
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --name";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { shell_call => $shell_call }});
 		
 		my ($output, $return_code) = $anvil->System->call({debug => 3, shell_call => $shell_call});

tools/anvil-manage-server

@@ -128,7 +128,7 @@ sub process_interactive
 		# Is this machine in an Anvil!?
 		$anvil->Database->get_hosts();
 		my $host_uuid                      = $anvil->Get->host_uuid();
-		my $anvil->data->{switches}{anvil} = $anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid}
+		   $anvil->data->{switches}{anvil} = $anvil->data->{hosts}{host_uuid}{$host_uuid}{anvil_uuid};
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			host_uuid         => $host_uuid,
 			"switches::anvil" => $anvil->data->{switches}{anvil},
@@ -1163,6 +1163,7 @@ sub interactive_configure_optical_disk
 		my $device_bus = $anvil->data->{optical_drives}{$index}{device_bus};
 		my $boot_order = $anvil->data->{optical_drives}{$index}{boot_order};
 		my $iso        = $anvil->data->{optical_drives}{$index}{iso};
+		my $changes    = 0;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			"s1:target"     => $target,
 			"s2:device_bus" => $device_bus,

tools/anvil-manage-server-storage

@@ -9,6 +9,15 @@
 # 
 # TODO: 
 # 
+# USAGE:
+# - Show
+#   - anvil-manage-server-storage --server srv01-fs37
+# - ISO
+#   - anvil-manage-server-storage --server srv01-fs37 --optical sda --insert /mnt/shared/files/CentOS-5.11-x86_64-bin-DVD-1of2.iso
+#   - anvil-manage-server-storage --server srv01-fs37 --optical sda --eject
+# - Disk
+#   - anvil-manage-server-storage --server srv01-fs37 --disk vdb --grow {+10GiB,150GiB}
+# 
 
 use strict;
 use warnings;
@@ -38,9 +47,13 @@ $anvil->Log->secure({set => 1});
 $anvil->Get->switches({list => [
 	"add",
 	"anvil", 
+	"disk", 
+	"eject", 
 	"grow",
+	"insert", 
+	"optical", 
 	"server",
-       	], man => $THIS_FILE});
+	], man => $THIS_FILE});
 $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}});
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
@@ -86,7 +99,21 @@ if (not $anvil->data->{switches}{server})
 	print "\nPlease specify which server you want to modify using '--server <name or uuid>'.\n\n";
 	$anvil->nice_exit({exit_code => 0});
 }
-manage_server($anvil);
+
+validate_server($anvil);
+
+if ($anvil->data->{switches}{optical})
+{
+	manage_optical($anvil);
+}
+elsif ($anvil->data->{switches}{disk})
+{
+	manage_disk($anvil);
+}
+else
+{
+	show_server_details($anvil);
+}
 
 $anvil->nice_exit({exit_code => 0});
 
@@ -95,58 +122,312 @@ $anvil->nice_exit({exit_code => 0});
 # Functions                                                                                                 #
 #############################################################################################################
 
-sub manage_server
+sub manage_disk
 {
 	my ($anvil) = @_;
 	
-	$anvil->Get->server_from_switch({
+	my $short_host_name = $anvil->Get->short_host_name;
-		debug      => 2, 
+	my $server_name     = $anvil->data->{switches}{server_name};
-		string     => $anvil->data->{switches}{server},
+	my $from_source     = get_definition_source($anvil);
-		anvil_uuid => $anvil->data->{switches}{anvil_uuid},
-	});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-		"switches::server_name" => $anvil->data->{switches}{server_name},
+		short_host_name => $short_host_name,
-		"switches::server_uuid" => $anvil->data->{switches}{server_uuid},
+		server_name     => $server_name, 
+		from_source     => $from_source, 
 	}});
+	if (not $anvil->data->{switches}{disk})
+	{
+		# If there's only one optical drive, select it automatically
+		my $count = keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{disk}{target}};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
+		if ($count == 1)
+		{
+			foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{disk}{target}})
+			{
+				$anvil->data->{switches}{disk} = $device_target;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					"switches::drive" => $anvil->data->{switches}{disk},
+				}});
+				last;
+			}
+		}
+		
+		if (not $anvil->data->{switches}{disk})
+		{
+			# Can't proceed.
+			print "\n[ Error ] - Please indicate the disk drive to work with using '--disk --drive <target_device>'.\n\n";
+			show_server_details($anvil);
+			$anvil->nice_exit({exit_code => 1});
+		}
+	}
 	
-	if (not $anvil->data->{switches}{server_uuid})
+	my $device_target = $anvil->data->{switches}{disk};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device_target => $device_target }});
+
+	my $device           = "disk";
+	my $alias            = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{alias};
+	my $boot_order       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{boot_order};
+	my $say_boot         = $boot_order eq "99" ? "--" : sprintf("%02d", $boot_order);
+	my $type             = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{type};
+	my $address_type     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{type};
+	my $address_bus      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{bus};
+	my $driver_name      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{name};
+	my $device_bus       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{device_bus};
+	my $driver_type      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{type};
+	my $address_domain   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{domain};
+	my $address_slot     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{slot};
+	my $address_function = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{function};
+	my $device_path      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path};
+	my $driver_io        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{io};
+	my $driver_cache     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{cache};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		's01:device_target'    => $device_target,
+		's02:alias'            => $alias, 
+		's03:boot_order'       => $boot_order,
+		's04:say_boot'         => $say_boot, 
+		's05:type'             => $type, 
+		's06:address_type'     => $address_type, 
+		's07:address_bus'      => $address_bus, 
+		's08:driver_name'      => $driver_name, 
+		's09:device_bus'       => $device_bus, 
+		's10:driver_type'      => $driver_type, 
+		's11:address_domain'   => $address_domain,
+		's12:address_slot'     => $address_slot, 
+		's13:address_function' => $address_function, 
+		's14:device_path'      => $device_path, 
+		's15:driver_io'        => $driver_io, 
+		's16:driver_cache'     => $driver_cache, 
+	}});
+	print "- Target: [".$device_target."], boot: [".$say_boot."], path: [".$device_path."], cache: [".$driver_cache."], driver type: [".$driver_type."]\n";
+	
+	my $volume = "";
+	
+	print "Sub-Nodes:\n";
+	my $drbd_resource = "";
+	foreach my $device_path (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{device}})
 	{
-		show_server_list($anvil);
+		my $on_lv         = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{on_lv};
-		if ($anvil->data->{switches}{anvil_uuid})
+		   $drbd_resource = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{resource};
+		my $device_target = $anvil->data->{server}{$short_host_name}{$server_name}{device}{$device_path}{target};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			's1:device_path'   => $device_path, 
+			's2:on_lv'         => $on_lv,
+			's3:drbd_resource' => $drbd_resource, 
+			's4:device_target' => $device_target, 
+		}});
+	}
+	foreach my $drbd_resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{drbd}{resource}})
+	{
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_resource => $drbd_resource }});
+		
+		# Get the DRBD volume data
+		load_drbd_data($anvil, $drbd_resource);
+	}
+	foreach my $host_type ("node", "dr")
+	{
+		foreach my $short_host_name (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}})
 		{
-			# Not found on the requested Anvil! node.
+			my $host_uuid = $anvil->data->{drbd_resource}{$drbd_resource}{host_type}{$host_type}{short_host_name}{$short_host_name}{host_uuid};
-			print "\nThe server: [".$anvil->data->{switches}{server}."] was not found on the Anvil! node: [".$anvil->data->{switches}{anvil_name}."]. Valid servers are above.\n\n";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				's1:short_host_name' => $short_host_name,
+				's2:host_uuid'       => $host_uuid, 
+			}});
+			print " |- Name: [".$short_host_name."], UUID: [".$host_uuid."]\n";
+			foreach my $volume_number (sort {$a cmp $b} keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}})
+			{
+				my $device_path  = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_path};
+				my $device_minor = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{device_minor};
+				my $volume_size  = $anvil->data->{drbd_resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{volume_size};
+				my $backing_disk = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{backing_disk};
+				my $meta_disk    = $anvil->data->{new}{resource}{$drbd_resource}{host_uuid}{$host_uuid}{volume_number}{$volume_number}{meta_disk};
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					's1:volume_number' => $volume_number,
+					's2:device_path'   => $device_path, 
+					's3:device_minor'  => $device_minor, 
+					's4:volume_size'   => $volume_size, 
+					's5:backing_disk'  => $backing_disk, 
+					's6:meta_disk'     => $meta_disk, 
+				}});
+				print "   ^- Volume: [".$volume_number."], backing device: [".$backing_disk."], DRBD minor: [".$device_minor."], size: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $volume_size})."]\n";
+			}
+		}
+	}
+	
+	# What are we doing?
+	if ($anvil->data->{switches}{grow})
+	{
+		### TODO: Make this work without the peer node being online.
+		# The server is allowed to be running, but both nodes and any DR hosts this is replicating to
+		# needs to be online.
+		
+		# Process the DRBD config to get the backing LVs.
+	}
+	
+	return(0);
+}
+
+sub manage_optical
+{
+	my ($anvil) = @_;
+	
+	my $short_host_name = $anvil->Get->short_host_name;
+	my $server_name     = $anvil->data->{switches}{server_name};
+	my $from_source     = get_definition_source($anvil);
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		short_host_name => $short_host_name,
+		server_name     => $server_name, 
+		from_source     => $from_source, 
+	}});
+	
+	if (not $anvil->data->{switches}{optical})
+	{
+		# If there's only one optical drive, select it automatically
+		my $count = keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}};
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { count => $count }});
+		if ($count == 1)
+		{
+			foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}})
+			{
+				$anvil->data->{switches}{optical} = $device_target;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+					"switches::drive" => $anvil->data->{switches}{optical},
+				}});
+				last;
+			}
+		}
+		
+		if (not $anvil->data->{switches}{optical})
+		{
+			# Can't proceed.
+			print "\n[ Error ] - Please indicate the optical drive to work with using '--optical <target_device>'.\n\n";
+			show_server_details($anvil);
+			$anvil->nice_exit({exit_code => 1});
+		}
+	}
+	my $device_target = $anvil->data->{switches}{optical};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device_target => $device_target }});
+	
+	my $eject_first        = 0;
+	my $alias              = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{alias};
+	my $boot_order         = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{boot_order};
+	my $say_boot           = $boot_order eq "99" ? "--" : sprintf("%02d", $boot_order);
+	my $type               = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{type};
+	my $address_type       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{type};
+	my $address_bus        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{bus};
+	my $driver_name        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{driver}{name};
+	my $device_bus         = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{device_bus};
+	my $driver_type        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{driver}{type};
+	my $address_controller = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{controller};
+	my $address_unit       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{unit};
+	my $address_target     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{address}{target};
+	my $device_path        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{cdrom}{target}{$device_target}{path};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		's01:device_target'      => $device_target,
+		's02:alias'              => $alias, 
+		's03:boot_order'         => $boot_order,
+		's04:say_boot'           => $say_boot, 
+		's05:type'               => $type, 
+		's06:address_type'       => $address_type, 
+		's07:address_bus'        => $address_bus, 
+		's08:driver_name'        => $driver_name, 
+		's09:device_bus'         => $device_bus, 
+		's10:driver_type'        => $driver_type, 
+		's11:address_controller' => $address_controller,
+		's12:address_unit'       => $address_unit, 
+		's13:address_target'     => $address_target, 
+		's14:device_path'        => $device_path, 
+	}});
+	
+	# Without a --source, the result is an eject.
+	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." change-media ".$server_name." --path ".$device_target." --update --live --config";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+	if ($anvil->data->{switches}{insert})
+	{
+		# Make sure the new target exists.
+		my $iso = $anvil->data->{switches}{insert};
+		if (not -f $iso)
+		{
+			print "[ Error ] - The target: [".$iso."] doesn't exist, can't insert it into the optical drive.\n";
+			update_definition($anvil, "undefine");
+			$anvil->nice_exit({exit_code => 1});
 		}
 		else
 		{
-			# Not found at all.
+			$shell_call .= " --source ".$anvil->data->{switches}{insert};
-			print "\nThe server: [".$anvil->data->{switches}{server}."] was not found. Valid servers are above.\n\n";
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 		}
-		$anvil->nice_exit({exit_code => 1});
 	}
 	
-	print "Working with the server: [".$anvil->data->{switches}{server_name}."], UUID: [".$anvil->data->{switches}{server_uuid}."]\n";
+	# If the server is running, update the on-disk and in-DB definition.
-	my $short_host_name   = $anvil->Get->short_host_name;
+	print "Defining the server: [".$server_name."] to prepare for 'virsh' modification of the server.\n";
-	my $server_name       = $anvil->data->{switches}{server_name};
+	update_definition($anvil, "define");
-	my $server_uuid       = $anvil->data->{switches}{server_uuid};
+	
-	my $server_definition = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml};
+	# Now we can modify the server using virsh.
+	if ($anvil->data->{switches}{insert})
+	{
+		print "- Inserting: [".$anvil->data->{switches}{insert}."] into the drive: [".$device_target."].\n";
+	}
+	else
+	{
+		print "- Ejecting: [".$anvil->data->{switches}{insert}."] from: [".$device_target."].\n";
+	}
+	my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
-		's1:short_host_name'   => $short_host_name, 
+		's1:output'      => $output,
-		's2:server_name'       => $server_name, 
+		's2:return_code' => $return_code, 
-		's3:server_uuid'       => $server_uuid, 
-		's4:server_definition' => $server_definition,
 	}});
+	print "'virsh' Output: [".$output."]\n";
 	
-	# Parse the definition.
+	print "Updating the stored definition and undefining the server now:\n";
-	$anvil->Server->parse_definition({
+	update_definition($anvil, "undefine");
-		debug      => 3,
+	print "Done!\n";
-		host       => $short_host_name,
+	
-		server     => $server_name, 
+	return(0);
-		source     => "from_virsh",
+}
-		definition => $server_definition, 
+
-	});
+sub get_definition_source
+{
+	my ($anvil) = @_;
+	
+	my $short_host_name = $anvil->Get->short_host_name;
+	my $server_name     = $anvil->data->{switches}{server_name};
+	my $from_source     = "";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		short_host_name => $short_host_name,
+		server_name     => $server_name, 
+		from_source     => $from_source, 
+	}});
+	if (exists $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh})
+	{
+		$from_source = "from_virsh";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
+	}
+	elsif (exists $anvil->data->{server}{$short_host_name}{$server_name}{from_disk})
+	{
+		$from_source = "from_disk";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
+	}
+	else
+	{
+		$from_source = "from_db";
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
+	}
+
+	return($from_source);
+}
+
+sub show_server_details
+{
+	my ($anvil) = @_;
+	
+	my $short_host_name = $anvil->Get->short_host_name;
+	my $server_name     = $anvil->data->{switches}{server_name};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		's1:short_host_name' => $short_host_name,
+		's2:server_name'     => $server_name, 
+	}});
 	
-	#$anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{boot_order}
+	my $from_source = get_definition_source($anvil);
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }});
 	foreach my $device ("disk", "cdrom")
 	{
 		if ($device eq "disk")
@@ -159,17 +440,17 @@ sub manage_server
 		}
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device => $device }});
 		next if $device ne "cdrom" &&  $device ne "disk";
-		foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}})
+		foreach my $device_target (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}})
 		{
-			my $alias        = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{alias};
+			my $alias        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{alias};
-			my $boot_order   = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{boot_order};
+			my $boot_order   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{boot_order};
 			my $say_boot     = $boot_order eq "99" ? "--" : sprintf("%02d", $boot_order);
-			my $type         = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{type};
+			my $type         = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{type};
-			my $address_type = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{type};
+			my $address_type = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{type};
-			my $address_bus  = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{bus};
+			my $address_bus  = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{bus};
-			my $driver_name  = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{driver}{name};
+			my $driver_name  = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{name};
-			my $device_bus   = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{device_bus};
+			my $device_bus   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{device_bus};
-			my $driver_type  = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{driver}{type};
+			my $driver_type  = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{type};
 			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 				's01:device_target' => $device_target,
 				's02:alias'         => $alias, 
@@ -184,12 +465,12 @@ sub manage_server
 			}});
 			if ($device eq "disk")
 			{
-				my $address_domain   = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{domain};
+				my $address_domain   = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{domain};
-				my $address_slot     = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{slot};
+				my $address_slot     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{slot};
-				my $address_function = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{function};
+				my $address_function = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{function};
-				my $device_path      = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{path};
+				my $device_path      = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path};
-				my $driver_io        = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{driver}{io};
+				my $driver_io        = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{io};
-				my $driver_cache     = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{driver}{cache};
+				my $driver_cache     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{driver}{cache};
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 					's1:address_domain'   => $address_domain,
 					's2:address_slot'     => $address_slot, 
@@ -202,10 +483,14 @@ sub manage_server
 			}
 			else
 			{
-				my $address_controller = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{controller};
+				my $address_controller = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{controller};
-				my $address_unit       = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{unit};
+				my $address_unit       = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{unit};
-				my $address_target     = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{address}{target};
+				my $address_target     = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{address}{target};
-				my $device_path        = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{device}{$device}{target}{$device_target}{path};
+				my $device_path        = "<empty>";
+				if (defined $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path})
+				{
+					$device_path = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$device}{target}{$device_target}{path};
+				}
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 					's1:address_controller' => $address_controller,
 					's2:address_unit'       => $address_unit, 
@@ -234,10 +519,29 @@ sub manage_server
 	foreach my $drbd_resource (sort {$a cmp $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{drbd}{resource}})
 	{
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_resource => $drbd_resource }});
+		
+		# Get the DRBD volume data
+		load_drbd_data($anvil, $drbd_resource);
+	}
+	
+	print "Sub-Nodes:\n";
+	show_volume($anvil, $drbd_resource, "node");
+	
+	my $dr_count = keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{dr}{short_host_name}};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_count => $dr_count }});
+	if ($dr_count)
+	{
+		print "DR Hosts:\n";
+		show_volume($anvil, $drbd_resource, "dr");
 	}
 	
-	# Get the DRBD volume data
+	return(0);
-	# Get the DRBD volume data
+}
+
+sub load_drbd_data
+{
+	my ($anvil, $drbd_resource) = @_;
+	
 	my $query = "
 SELECT 
     a.host_uuid, 
@@ -302,17 +606,6 @@ AND
 		}
 	}
 	
-	print "Sub-Nodes:\n";
-	show_volume($anvil, $drbd_resource, "node");
-	
-	my $dr_count = keys %{$anvil->data->{drbd_resource}{$drbd_resource}{host_type}{dr}{short_host_name}};
-	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_count => $dr_count }});
-	if ($dr_count)
-	{
-		print "DR Hosts:\n";
-		show_volume($anvil, $drbd_resource, "dr");
-	}
-	
 	return(0);
 }
 
@@ -350,6 +643,272 @@ sub show_volume
 	return(0);
 }
 
+sub validate_server
+{
+	my ($anvil) = @_;
+	
+	$anvil->Get->server_from_switch({
+		debug      => 2, 
+		string     => $anvil->data->{switches}{server},
+		anvil_uuid => $anvil->data->{switches}{anvil_uuid},
+	});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		"switches::server_name" => $anvil->data->{switches}{server_name},
+		"switches::server_uuid" => $anvil->data->{switches}{server_uuid},
+	}});
+	
+	if (not $anvil->data->{switches}{server_uuid})
+	{
+		show_server_list($anvil);
+		if ($anvil->data->{switches}{anvil_uuid})
+		{
+			# Not found on the requested Anvil! node.
+			print "\nThe server: [".$anvil->data->{switches}{server}."] was not found on the Anvil! node: [".$anvil->data->{switches}{anvil_name}."]. Valid servers are above.\n\n";
+		}
+		else
+		{
+			# Not found at all.
+			print "\nThe server: [".$anvil->data->{switches}{server}."] was not found. Valid servers are above.\n\n";
+		}
+		$anvil->nice_exit({exit_code => 1});
+	}
+	
+	print "Working with the server: [".$anvil->data->{switches}{server_name}."], UUID: [".$anvil->data->{switches}{server_uuid}."]\n";
+	my $short_host_name   = $anvil->Get->short_host_name;
+	my $server_name       = $anvil->data->{switches}{server_name};
+	my $server_uuid       = $anvil->data->{switches}{server_uuid};
+	my $server_definition = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml};
+	my $server_host_uuid  = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
+	my $server_state      = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		's1:short_host_name'   => $short_host_name, 
+		's2:server_name'       => $server_name, 
+		's3:server_uuid'       => $server_uuid, 
+		's4:server_definition' => $server_definition,
+		's5:server_host_uuid'  => $server_host_uuid, 
+		's6:server_state'      => $server_state, 
+	}});
+	
+	# Parse the definition.
+	$anvil->Server->parse_definition({
+		debug      => 3,
+		host       => $short_host_name,
+		server     => $server_name, 
+		source     => "from_db",
+		definition => $server_definition, 
+	});
+	
+	# Can we read the XML definition?
+	$anvil->Server->get_status({
+		debug  => 2,
+		server => $server_name,
+		host   => $short_host_name,
+	});
+	
+	if (not $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{xml})
+	{
+		# The server isn't actually running... Not here anyway.
+		if ($server_state eq "running")
+		{
+			my $server_host_name = $anvil->Get->get_host_from_uuid({
+				short     => 1,
+				host_uuid => $server_host_uuid,
+			});
+			
+			print "The server: [".$server_name."] appears to be running on: [".$server_host_name."], please run this on that host.\n";
+			$anvil->nice_exit({exit_code => 1});
+		}
+	}
+	
+	return(0);
+}
+
+# Update the definition on disk and in the DB, and define or undefine if requested.
+sub update_definition
+{
+	my ($anvil, $task) = @_;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { task => $task }});
+	
+	my $short_host_name     = $anvil->Get->short_host_name;
+	my $server_name         = $anvil->data->{switches}{server_name};
+	my $server_uuid         = $anvil->data->{switches}{server_uuid};
+	my $server_definition   = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml};
+	my $server_host_uuid    = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid};
+	my $server_state        = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state};
+	my $definition_file     = $anvil->data->{path}{directories}{shared}{definitions}."/".$server_name.".xml";
+	my $server_running_here = 0;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		's1:short_host_name'   => $short_host_name, 
+		's2:server_name'       => $server_name, 
+		's3:server_uuid'       => $server_uuid, 
+		's4:server_definition' => $server_definition,
+		's5:server_host_uuid'  => $server_host_uuid, 
+		's6:server_state'      => $server_state, 
+		's7:definition_file'   => $definition_file, 
+	}});
+	
+	# See if the server is running locally
+	my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --name";
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+	
+	my ($output, $return_code) = $anvil->System->call({debug => 2, shell_call => $shell_call});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		's1:output'      => $output,
+		's2:return_code' => $return_code, 
+	}});
+	foreach my $this_server (split/\n/, $output)
+	{
+		$this_server = $anvil->Words->clean_spaces({string => $this_server});
+		next if not $this_server;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { this_server => $this_server }});
+		
+		if ($this_server eq $server_name)
+		{
+			$server_running_here = 1;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_running_here => $server_running_here }});
+			last;
+		}
+	}
+	
+	my $disk_definition  = $anvil->data->{server}{$short_host_name}{$server_name}{from_disk}{xml}  ? $anvil->data->{server}{$short_host_name}{$server_name}{from_disk}{xml}  : "";
+	my $virsh_definition = $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{xml} ? $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{xml} : "";
+	my $use_definition   = "";
+	if (($server_running_here) or (($server_state eq "running") && ($virsh_definition)))
+	{
+		# Get the live definition
+		if ($server_running_here)
+		{
+			my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml --inactive ".$server_name;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+			
+			my ($live_virsh_definition, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+				live_virsh_definition => $live_virsh_definition, 
+				return_code           => $return_code,
+			}});
+			
+			my ($problem) = $anvil->Server->parse_definition({
+				server     => $server_name,
+				source     => "from_virsh", 
+				definition => $live_virsh_definition, 
+			});
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
+			if (not $problem)
+			{
+				$use_definition = $live_virsh_definition;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
+				
+				$anvil->Server->parse_definition({
+					debug      => 3,
+					host       => $short_host_name,
+					server     => $server_name, 
+					source     => "from_virsh",
+					definition => $live_virsh_definition, 
+				});
+			}
+			else
+			{
+				$use_definition = $virsh_definition;
+				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
+				
+				$anvil->Server->parse_definition({
+					debug      => 3,
+					host       => $short_host_name,
+					server     => $server_name, 
+					source     => "from_virsh",
+					definition => $virsh_definition, 
+				});
+			}
+		}
+		else
+		{
+			$use_definition = $virsh_definition;
+			$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
+			
+			$anvil->Server->parse_definition({
+				debug      => 3,
+				host       => $short_host_name,
+				server     => $server_name, 
+				source     => "from_virsh",
+				definition => $virsh_definition, 
+			});
+		}
+	}
+	else
+	{
+		$use_definition = $disk_definition;
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { use_definition => $use_definition }});
+		
+		$anvil->Server->parse_definition({
+			debug      => 3,
+			host       => $short_host_name,
+			server     => $server_name, 
+			source     => "from_disk",
+			definition => $virsh_definition, 
+		});
+	}
+	
+	if (not $use_definition)
+	{
+		# What?!
+		print "[ Error ] - Failed to find an on disk or from virsh definition for the server: [".$server_name."]. Unable to proceed.\n";
+		$anvil->nice_exit({exit_code => 1});
+	}
+	
+	# Update the stored definition
+	$anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml} = $use_definition;
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+		"servers::server_uuid::${server_uuid}::server_definition_xml" => $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml},
+	}});
+	
+	# Update the on-disk definition
+	my ($failed) = $anvil->Storage->write_file({
+		secure    => 1, 
+		file      => $definition_file, 
+		body      => $use_definition, 
+		overwrite => 1,
+	});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { failed => $failed }});
+	my $server_definition_uuid = $anvil->Database->insert_or_update_server_definitions({
+		debug                         => 2,
+		server_definition_xml         => $use_definition,  
+		server_definition_server_uuid => $server_uuid, 
+	});
+	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => { server_definition_uuid => $server_definition_uuid }});
+	
+	if ($task eq "define")
+	{
+		# Define the server.
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." define ".$definition_file,
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		
+		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call, source => $THIS_FILE, line => __LINE__});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			output      => $output, 
+			return_code => $return_code,
+		}});
+	}
+	elsif ($task eq "undefine")
+	{
+		# Undefine the server.
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server_name,
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
+		
+		my ($output, $return_code) = $anvil->System->call({
+			debug      => 2,
+			shell_call => $shell_call, 
+			source     => $THIS_FILE, 
+			line       => __LINE__,
+		});
+		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
+			output      => $output, 
+			return_code => $return_code,
+		}});
+	}
+	
+	return(0);
+}
+
 sub show_server_list
 {
 	my ($anvil) = @_;

tools/anvil-provision-server

@@ -590,7 +590,7 @@ sub provision_server
 	my $status     = "";
 	while($waiting)
 	{
-		my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." list --all"});
+		my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." list --all"});
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 			output      => $output,
 			return_code => $return_code,
@@ -611,7 +611,7 @@ sub provision_server
 				elsif ($status eq "shut off")
 				{
 					# Try to boot it.
-					my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." start ".$server});
+					my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." start ".$server});
 					$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 						output      => $output,
 						return_code => $return_code,
@@ -641,7 +641,7 @@ sub provision_server
 	$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 2, key => "job_0201"});
 	
 	# Dump the server's XML.
-	(my $server_definition_xml, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." dumpxml ".$server});
+	(my $server_definition_xml, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." dumpxml ".$server});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		server_definition_xml => $server_definition_xml,
 		return_code           => $return_code,
@@ -699,7 +699,7 @@ sub provision_server
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_definition_uuid => $server_definition_uuid }});
 	
 	# Undefine the server
-	(my $output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." undefine ".$server});
+	(my $output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$server});
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		output      => $output,
 		return_code => $return_code,
@@ -2838,11 +2838,12 @@ sub interactive_ask_server_confirm
 	# terminal isn't set with --ci-test set.
 	check_anvil($anvil);
 	
-	$anvil->Database->get_anvils();
+	$anvil->Database->get_hosts({debug => 3});
-	$anvil->Database->get_files();
+	$anvil->Database->get_anvils({debug => 3});
-	$anvil->Database->get_file_locations();
+	$anvil->Database->get_files({debug => 2});
-	$anvil->Get->virsh_list_os();
+	$anvil->Database->get_file_locations({debug => 2});
-
+	$anvil->Get->virsh_list_os({debug => 3});
+	
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
 		"new_server::anvil_uuid" => $anvil->data->{new_server}{anvil_uuid},
 		"switches::ci-test"      => $anvil->data->{switches}{'ci-test'},
@@ -3234,6 +3235,7 @@ sub interactive_ask_server_confirm
 						free_space => $say_max_storage_group_size,
 					}})."\n";
 				}
+				
 				print $anvil->Words->string({key => "job_0460"})."\n";
 				foreach my $file_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{file_name}})
 				{
@@ -3245,7 +3247,7 @@ sub interactive_ask_server_confirm
 					print $anvil->Words->string({key => "job_0461", variables => {
 						name => $file_name,
 						uuid => $file_uuid,
-						size => $say_max_storage_group_size,
+						size => $say_size,
 					}})."\n";
 				}
 				print $anvil->Words->string({key => "job_0462"})."\n";

tools/anvil-rename-server

@@ -579,7 +579,7 @@ sub rename_server
 		
 		# Make a 'virsh undefine <server>' just in case the old name is still defined. We don't care 
 		# if this succeeds or fails as the server should not be defined anyway.
-		my $shell_call = $anvil->data->{path}{exe}{virsh}." undefine ".$old_server_name;
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." undefine ".$old_server_name;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 		
 		my $output      = "";

tools/anvil-update-states

@@ -147,7 +147,7 @@ sub update_network
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { host_type => $host_type }});
 	if (($host_type eq "node") or ($host_type eq "dr"))
 	{
-		my $shell_call = $anvil->data->{path}{exe}{virsh}." net-list --all --name";
+		my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-list --all --name";
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 		
 		my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
@@ -168,7 +168,7 @@ sub update_network
 				
 				$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, 'print' => 1, key => "striker_0287", variables => { bridge => $line }});
 				
-				my $shell_call = $anvil->data->{path}{exe}{virsh}." net-destroy ".$line;
+				my $shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-destroy ".$line;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 				my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 
@@ -176,7 +176,7 @@ sub update_network
 					return_code => $return_code, 
 				}});
 				
-				$shell_call = $anvil->data->{path}{exe}{virsh}." net-undefine ".$line;
+				$shell_call = $anvil->data->{path}{exe}{setsid}." --wait ".$anvil->data->{path}{exe}{virsh}." net-undefine ".$line;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
 				($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 

tools/striker-prep-database

@@ -35,9 +35,9 @@ $| = 1;
 
 my $anvil = Anvil::Tools->new();
 
-$anvil->Get->switches;
 $anvil->Log->level({set => 2});
 $anvil->Log->secure({set => 1});
+$anvil->Get->switches;
 $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }});
 
 $anvil->System->_check_anvil_conf({debug => 2});