Added a 1 second delay to nmcli calls

* Also fixed a bug Database->get_storage_group_data() to add a missing
  column to adding members.

Signed-off-by: digimer <mkelly@alteeve.ca>

Anvil/Tools/Cluster.pm

@@ -1251,7 +1251,7 @@ sub check_stonith_config
 				pcs_add_command => $pcs_add_command =~ /passw/ ? $anvil->Log->is_secure($pcs_add_command) : $pcs_add_command,
 			}});
 			
-			# If there's an entry in the CIB, so if it's different somehow
+			# If there's an entry in the CIB, see if it's different somehow
 			if (exists $anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name})
 			{
 				foreach my $argument (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{node}{$node_name}{fencing}{device}{$ipmi_stonith_name}{argument}})

Anvil/Tools/Database.pm

@@ -6473,12 +6473,14 @@ INSERT INTO
     storage_group_member_storage_group_uuid, 
     storage_group_member_host_uuid, 
     storage_group_member_vg_uuid, 
+    storage_group_member_note, 
     modified_date 
 ) VALUES (
     ".$anvil->Database->quote($storage_group_member_uuid).", 
     ".$anvil->Database->quote($storage_group_uuid).", 
     ".$anvil->Database->quote($this_host_uuid).", 
     ".$anvil->Database->quote($closest_scan_lvm_vg_uuid).", 
+    'auto-created',
     ".$anvil->Database->quote($anvil->Database->refresh_timestamp)."
 );";
 						$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { query => $query }});

Anvil/Tools/Network.pm

@@ -1146,6 +1146,8 @@ sub collect_data
 					output      => $output,
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 			}
 		}
 	}
@@ -1197,6 +1199,8 @@ sub collect_data
 					output      => $output,
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 				
 				$rescan = 1;
 				$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { rescan => $rescan }});
@@ -4107,6 +4111,8 @@ sub modify_connection
 		output      => $output,
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	return($output, $return_code);
 }
@@ -4589,6 +4595,8 @@ sub reset_connection
 		output      => $output,
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	$shell_call = $anvil->data->{path}{exe}{nmcli}." connection up ".$uuid;
 	$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@@ -4597,6 +4605,8 @@ sub reset_connection
 		output      => $output,
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	return($output, $return_code);
 }

tools/anvil-configure-host

@@ -799,6 +799,8 @@ sub reconfigure_bridges
 					output      => $output,
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 				
 				if ($return_code)
 				{
@@ -848,6 +850,8 @@ sub reconfigure_bridges
 						output      => $output,
 						return_code => $return_code, 
 					}});
+					# NM seems to have a race issue, so we sleep a second after nmcli calls.
+					sleep 1;
 				}
 				
 				# Rescan.
@@ -968,6 +972,8 @@ sub reconfigure_bridges
 				output      => $output,
 				return_code => $return_code, 
 			}});
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			sleep 1;
 			
 			if ($return_code)
 			{
@@ -1173,6 +1179,8 @@ sub reconfigure_bonds
 					output      => $output,
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 				
 				if ($return_code)
 				{
@@ -1222,6 +1230,8 @@ sub reconfigure_bonds
 						output      => $output,
 						return_code => $return_code, 
 					}});
+					# NM seems to have a race issue, so we sleep a second after nmcli calls.
+					sleep 1;
 				}
 				
 				# Done! Rescanning the network config
@@ -1355,6 +1365,8 @@ sub reconfigure_bonds
 					output      => $output,
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 				
 				if ($return_code)
 				{
@@ -1758,6 +1770,8 @@ sub reconfigure_ip_addresses
 				output      => $output,
 				return_code => $return_code, 
 			}});
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			sleep 1;
 			
 			# Restart the interface
 			$anvil->Job->update_progress({
@@ -1838,6 +1852,8 @@ sub reconfigure_interfaces
 					output      => $output, 
 					return_code => $return_code, 
 				}});
+				# NM seems to have a race issue, so we sleep a second after nmcli calls.
+				sleep 1;
 				
 				# Rescan.
 				$anvil->Job->update_progress({
@@ -2227,6 +2243,8 @@ sub rename_interface
 			output      => $output, 
 			return_code => $return_code, 
 		}});
+		# NM seems to have a race issue, so we sleep a second after nmcli calls.
+		sleep 1;
 		
 		$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values connection.interface-name connection show ".$nm_uuid;
 		$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
@@ -2298,6 +2316,8 @@ sub rename_interface
 		output      => $output, 
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	# Read it back
 	$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values match.interface-name connection show ".$nm_uuid;
@@ -2347,6 +2367,8 @@ sub rename_interface
 		output      => $output, 
 		return_code => $return_code, 
 	}});
+	# NM seems to have a race issue, so we sleep a second after nmcli calls.
+	sleep 1;
 	
 	# Read it back
 	$shell_call = $anvil->data->{path}{exe}{nmcli}." --get-values connection.id connection show ".$nm_uuid;

tools/anvil-join-anvil

@@ -2841,14 +2841,20 @@ sub check_local_network
 			output      => $output,
 			return_code => $return_code, 
 		}});
+		# NM seems to have a race issue, so we sleep a second after nmcli calls.
+		sleep 1;
 		
 		foreach my $interface_name (sort {$a cmp $b} @{$restart_interfaces})
 		{
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0462", variables => { interface => $interface_name }});
 			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection down \"".$interface_name."\""});
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			sleep 1;
 			
 			$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, 'print' => 1, key => "log_0464", variables => { interface => $interface_name }});
 			$anvil->System->call({debug => 3, shell_call => $anvil->data->{path}{exe}{nmcli}." connection up \"".$interface_name."\""});
+			# NM seems to have a race issue, so we sleep a second after nmcli calls.
+			sleep 1;
 		}
 		
 		# Wait for a DB connection. We'll wait up to 130 seconds (updelay is 120 seconds, plus a small buffer).