fix(cgi-bin): draft pipe repair without destruction

cgi-bin/manage_vnc_pipes

@@ -132,11 +132,12 @@ sub is_websockify_exists
 {
 	my $parameters      = shift;
 	my $server_uuid     = $parameters->{server_uuid};
+	my $host_uuid       = $parameters->{host_uuid};
 	my $server_vnc_port = $parameters->{server_vnc_port};
 
 	my $query = "
 SELECT
-	vnc.server_vnc_port, hos.host_name, vnc.ws_pid, vnc.ssh_tunnel_forward_port
+	vnc.server_vnc_port, hos.host_name, hos.host_uuid, vnc.ws_pid, vnc.ws_source_port
 FROM
 	public.vnc_pipes AS vnc
 JOIN
@@ -149,87 +150,126 @@ WHERE
 
 	my $results        = $anvil->Database->query({ query => $query, source => $THIS_FILE, line => __LINE__ });
 	my $count          = @{$results};
-	my $existing_websockify;
+	my $ws_exists_info = { exists_code => 0 };
 
 	if ($count > 0)
 	{
 		my $row                       = $results->[0];
 		my $server_vnc_port_in_record = $row->[0];
 		my $host_name                 = $row->[1];
-		my $ws_pid                    = $row->[2];
+		my $host_uuid_in_record       = $row->[2];
-		my $ssh_tunnel_forward_port   = $row->[3];
+		my $ws_pid                    = $row->[3];
+		my $ws_source_port            = $row->[4];
 		my $clean_up_parameters       = { host_name => $host_name, ws_pid => $ws_pid };
 
-		if ($server_vnc_port != $server_vnc_port_in_record)
+		$ws_exists_info->{ws_pid}         = $ws_pid;
-		{
+		$ws_exists_info->{ws_source_port} = $ws_source_port;
-			# VNC server port mismatch/oudated; require clean up.
+		$ws_exists_info->{exists_code}    = 1;
 
+		if ($host_uuid != $host_uuid_in_record)
+		{
+			# VNC server host mismatch; try to stop the recorded instance.
+			# Likely happens after a server migration.
 			stop_websockify($clean_up_parameters);
-			stop_related($clean_up_parameters);
-			delete_vnc_pipe($clean_up_parameters);
 
-			return;
+			# No need to preserve the websockify source port in
+			# this case because the tunnel will need to be replaced
+			# as well.
+			delete $ws_exists_info->{ws_source_port};
+
+			return $ws_exists_info;
 		}
 
-		if (not is_websockify_process($clean_up_parameters))
+		if ($server_vnc_port != $server_vnc_port_in_record)
 		{
-			# Process died; require clean up.
+			# VNC server port mismatch; try to stop the recorded instance.
+			stop_websockify($clean_up_parameters);
 
-			stop_related($clean_up_parameters);
+			return $ws_exists_info;
-			delete_vnc_pipe($clean_up_parameters);
+		}
 
-			return;
+		if (not is_websockify_process($clean_up_parameters))
+		{
+			# The recorded instance died.
+			return $ws_exists_info;
 		}
 
 		# Passed all tests; process considered exists.
-		$existing_websockify = { ws_pid => $ws_pid, ssh_tunnel_forward_port => $ssh_tunnel_forward_port };
+		$ws_exists_info->{exists_code} = 2;
 	}
 
-	return $existing_websockify;
+	return $ws_exists_info;
 }
 
 sub is_ssh_tunnel_exists
 {
 	my $parameters           = shift;
 	my $server_uuid          = $parameters->{server_uuid};
-	my $host_uuid   = $parameters->{host_uuid};
+	my $ssh_tunnel_host_uuid = $parameters->{host_uuid};
+	my $ws_host_uuid         = $parameters->{ws_host_uuid};
+	my $ws_source_port       = $parameters->{ws_source_port};
 
 	my $query = "
 SELECT
-	ssh_tunnel_pid
+	ws_host_uuid, ws_source_port, ssh_tunnel_pid, ssh_tunnel_forward_port
 FROM
 	public.vnc_pipes
 WHERE
 	server_uuid = ".$anvil->Database->quote($server_uuid)."
 AND
-	ssh_tunnel_host_uuid = ".$anvil->Database->quote($host_uuid)."
+	ssh_tunnel_host_uuid = ".$anvil->Database->quote($ssh_tunnel_host_uuid)."
 ;";
 
 	my $results                = $anvil->Database->query({ query => $query, source => $THIS_FILE, line => __LINE__ });
 	my $count                  = @{$results};
-	my $existing_ssh_tunnel;
+	my $ssh_tunnel_exists_info = { exists_code => 0 };
 
 	if ($count == 1)
 	{
 		my $row                      = $results->[0];
-		my $ssh_tunnel_pid      = $row->[0];
+		my $ws_host_uuid_in_record   = $row->[0];
+		my $ws_source_port_in_record = $row->[1];
+		my $ssh_tunnel_pid           = $row->[2];
+		my $ssh_tunnel_forward_port  = $row->[3];
 		my $clean_up_parameters      = { ssh_tunnel_pid => $ssh_tunnel_pid };
 
-		if (not is_ssh_process($clean_up_parameters))
+		$ssh_tunnel_exists_info->{ssh_tunnel_pid}          = $ssh_tunnel_pid;
+		$ssh_tunnel_exists_info->{ssh_tunnel_forward_port} = $ssh_tunnel_forward_port;
+		$ssh_tunnel_exists_info->{exists_code}             = 1;
+
+		if ($ws_host_uuid != $ws_host_uuid_in_record)
 		{
-			# Process died; require clean up.
+			# Websockify host mismatch; try to stop the recorded instance.
+			# Likely happens after a server migration.
+			stop_ssh_tunnel($clean_up_parameters);
 
-			stop_related($clean_up_parameters);
+			# No need to preserve the SSH tunnel forward port in
-			delete_vnc_pipe($clean_up_parameters);
+			# this case because the websockify instance will need
+			# to be replaced as well.
+			delete $ssh_tunnel_exists_info->{ssh_tunnel_forward_port};
 
-			return;
+			return $ssh_tunnel_exists_info;
+		}
+
+		if ($ws_source_port != $ws_source_port_in_record)
+		{
+			# Websockify source port mismatch; try to stop the recorded instance.
+			stop_ssh_tunnel($clean_up_parameters);
+
+			return $ssh_tunnel_exists_info;
+		}
+
+		if (not is_ssh_process($clean_up_parameters))
+		{
+			# The recorded tunnel died.
+			return $ssh_tunnel_exists_info;
 		}
 
 		# Passed all tests; tunnel considered exists.
-		$existing_ssh_tunnel = { ssh_tunnel_pid => $ssh_tunnel_pid };
+		$ssh_tunnel_exists_info->{exists_code} = 2;
 	}
 
-	return $existing_ssh_tunnel;
+	return $ssh_tunnel_exists_info;
 }
 
 sub start_websockify
@@ -237,22 +277,39 @@ sub start_websockify
 	my $parameters  = shift;
 	my $server_uuid = $parameters->{server_uuid};
 	my $host_name   = $parameters->{host_name};
+	my $host_uuid   = $parameters->{host_uuid};
 	my $target_port = $parameters->{target_port};
+	my $source_port = $parameters->{source_port};
 	my $ws_info;
 
-	my $existing_websockify = is_websockify_exists({ server_uuid => $server_uuid, server_vnc_port => $target_port });
+	my $ws_exists_info = is_websockify_exists({
+		server_uuid     => $server_uuid,
+		host_uuid       => $host_uuid,
+		server_vnc_port => $target_port
+	});
 
-	if (defined $existing_websockify)
+	if ($ws_exists_info->{exists_code} == 2)
 	{
 		$ws_info                = {};
-		$ws_info->{pid}         = $existing_websockify->{ws_pid};
+		$ws_info->{pid}         = $ws_exists_info->{ws_pid};
-		$ws_info->{source_port} = $existing_websockify->{ssh_tunnel_forward_port};
+		$ws_info->{source_port} = $ws_exists_info->{ws_source_port};
-		$ws_info->{is_existing} = 1;
+	}
+	else
+	{
+		if (not defined $source_port)
+		{
+			if (defined $ws_exists_info->{ws_source_port})
+			{
+				$source_port = $ws_exists_info->{ws_source_port};
 			}
 			else
 			{
 				my $source_port_base = 10000;
-		my $source_port      = $source_port_base + $target_port;
+
+				$source_port = $source_port_base + $target_port;
+			}
+		}
+
 		my $shell_call = "ssh -n ".$host_name." 'websockify ".$source_port." :".$target_port." &>/dev/null & echo pid:\$!'";
 
 		my ($shell_output, $shell_return_code) = $anvil->System->call({ shell_call => $shell_call });
@@ -269,11 +326,23 @@ sub start_websockify
 			$ws_info                = {};
 			$ws_info->{pid}         = $ws_pid;
 			$ws_info->{source_port} = $source_port;
+			$ws_info->{is_new}      = 1;
 
 			$anvil->Log->variables({ source => $THIS_FILE, line => __LINE__, level => 2, list => {
 				ws_pid         => $ws_pid,
 				ws_source_port => $source_port
 			} });
+
+			if ($ws_exists_info->{exists_code} == 1)
+			{
+				update_vnc_pipe({
+					server_uuid     => $server_uuid,
+					server_vnc_port => $target_port,
+					ws_host_uuid    => $host_uuid,
+					ws_pid          => $ws_pid,
+					ws_source_port  => $source_port
+				});
+			}
 		}
 	}
 
@@ -288,6 +357,8 @@ sub stop_websockify
 
 	if (is_websockify_process($parameters))
 	{
+		# TODO: make sure no other tunnel is using the same websockify instance.
+
 		my $shell_call = "ssh -n ".$host_name." \"kill ".$ws_pid."\"";
 		my $shell_output;
 		my $shell_return_code;
@@ -321,20 +392,39 @@ sub start_ssh_tunnel
 	my $server_uuid             = $parameters->{server_uuid};
 	my $host_uuid               = $parameters->{host_uuid};
 	my $ws_host_name            = $parameters->{ws_host_name};
+	my $ws_host_uuid            = $parameters->{ws_host_uuid};
 	my $ws_source_port          = $parameters->{ws_source_port};
+	my $ssh_tunnel_forward_port = $parameters->{ssh_tunnel_forward_port};
 	my $ssh_tunnel_info;
 
-	my $existing_ssh_tunnel = is_ssh_tunnel_exists({ server_uuid => $server_uuid, host_uuid => $host_uuid });
+	my $ssh_tunnel_exists_info = is_ssh_tunnel_exists({
+		server_uuid          => $server_uuid,
+		ssh_tunnel_host_uuid => $host_uuid,
+		ws_host_uuid         => $ws_host_uuid,
+		ws_source_port       => $ws_source_port
+	});
 
-	if (defined $existing_ssh_tunnel)
+	if ($ssh_tunnel_exists_info->{exists_code} == 2)
 	{
 		$ssh_tunnel_info                 = {};
-		$ssh_tunnel_info->{pid}         = $existing_ssh_tunnel->{ssh_tunnel_pid};
+		$ssh_tunnel_info->{pid}          = $ssh_tunnel_exists_info->{ssh_tunnel_pid};
-		$ssh_tunnel_info->{is_existing} = 1;
+		$ssh_tunnel_info->{forward_port} = $ssh_tunnel_exists_info->{ssh_tunnel_forward_port};
+	}
+	else
+	{
+		if (not defined $ssh_tunnel_forward_port)
+		{
+			if (defined $ssh_tunnel_exists_info->{ssh_tunnel_forward_port})
+			{
+				$ssh_tunnel_forward_port = $ssh_tunnel_exists_info->{ssh_tunnel_forward_port};
 			}
 			else
 			{
-		my $shell_call = "ssh -nN -L ".$ws_source_port.":localhost:".$ws_source_port." ".$ws_host_name." & echo pid:\$!";
+				$ssh_tunnel_forward_port = $ws_source_port;
+			}
+		}
+
+		my $shell_call = "ssh -nN -L ".$ssh_tunnel_forward_port.":localhost:".$ws_source_port." ".$ws_host_name." & echo pid:\$!";
 
 		my ($shell_output, $shell_return_code) = $anvil->System->call({ shell_call => $shell_call });
 		$anvil->Log->variables({ source => $THIS_FILE, line => __LINE__, level => 2, list => {
@@ -349,8 +439,20 @@ sub start_ssh_tunnel
 
 			$ssh_tunnel_info                 = {};
 			$ssh_tunnel_info->{pid}          = $ssh_tunnel_pid;
+			$ssh_tunnel_info->{forward_port} = $ssh_tunnel_forward_port;
+			$ssh_tunnel_info->{is_new}       = 1;
 
 			$anvil->Log->variables({ source => $THIS_FILE, line => __LINE__, level => 2, list => { ssh_tunnel_pid => $ssh_tunnel_pid } });
+
+			if ($ssh_tunnel_exists_info->{exists_code} == 1)
+			{
+				update_vnc_pipe({
+					server_uuid             => $server_uuid,
+					ssh_tunnel_host_uuid    => $host_uuid,
+					ssh_tunnel_pid          => $ssh_tunnel_pid,
+					ssh_tunnel_forward_port => $ssh_tunnel_forward_port
+				});
+			}
 		}
 	}
 
@@ -452,6 +554,7 @@ CREATE TABLE IF NOT EXISTS public.vnc_pipes (
 	server_vnc_port		numeric				not null,
 	ws_host_uuid		uuid				not null,
 	ws_pid			numeric				not null,
+	ws_source_port		numeric				not null,
 	ssh_tunnel_host_uuid	uuid				not null,
 	ssh_tunnel_pid		numeric				not null,
 	ssh_tunnel_forward_port	numeric				not null,
@@ -475,6 +578,7 @@ sub insert_vnc_pipe
 	my $server_vnc_port         = $parameters->{server_vnc_port};
 	my $ws_host_uuid            = $parameters->{ws_host_uuid};
 	my $ws_pid                  = $parameters->{ws_pid};
+	my $ws_source_port          = $parameters->{ws_source_port};
 	my $ssh_tunnel_host_uuid    = $parameters->{ssh_tunnel_host_uuid};
 	my $ssh_tunnel_pid          = $parameters->{ssh_tunnel_pid};
 	my $ssh_tunnel_forward_port = $parameters->{ssh_tunnel_forward_port};
@@ -489,6 +593,7 @@ INSERT INTO public.vnc_pipes (
 	server_vnc_port,
 	ws_host_uuid,
 	ws_pid,
+	ws_source_port,
 	ssh_tunnel_host_uuid,
 	ssh_tunnel_pid,
 	ssh_tunnel_forward_port,
@@ -499,6 +604,7 @@ INSERT INTO public.vnc_pipes (
 	".$anvil->Database->quote($server_vnc_port).",
 	".$anvil->Database->quote($ws_host_uuid).",
 	".$anvil->Database->quote($ws_pid).",
+	".$anvil->Database->quote($ws_source_port).",
 	".$anvil->Database->quote($ssh_tunnel_host_uuid).",
 	".$anvil->Database->quote($ssh_tunnel_pid).",
 	".$anvil->Database->quote($ssh_tunnel_forward_port).",
@@ -508,6 +614,48 @@ INSERT INTO public.vnc_pipes (
 	$anvil->Database->write({ query => $query, source => $THIS_FILE, line => __LINE__ });
 }
 
+sub update_vnc_pipe
+{
+	my $parameters              = shift;
+	my $server_uuid             = $parameters->{server_uuid};
+	my $server_vnc_port         = $parameters->{server_vnc_port};
+	my $ws_host_uuid            = $parameters->{ws_host_uuid};
+	my $ws_pid                  = $parameters->{ws_pid};
+	my $ws_source_port          = $parameters->{ws_source_port};
+	my $ssh_tunnel_host_uuid    = $parameters->{ssh_tunnel_host_uuid};
+	my $ssh_tunnel_pid          = $parameters->{ssh_tunnel_pid};
+	my $ssh_tunnel_forward_port = $parameters->{ssh_tunnel_forward_port};
+	my $set_string;
+	my $condition_string        = "server_uuid = ".$anvil->Database->quote($server_uuid);
+
+	if ((defined $ws_host_uuid) and (defined $ws_pid) and (defined $ws_source_port))
+	{
+		$set_string = "
+server_vnc_port = ".$anvil->Database->quote($server_vnc_port).",
+ws_host_uuid    = ".$anvil->Database->quote($ws_host_uuid).",
+ws_pid          = ".$anvil->Database->quote($ws_pid).",
+ws_source_port  = ".$anvil->Database->quote($ws_source_port)."
+";
+	}
+	elsif ((defined $ssh_tunnel_host_uuid) and (defined $ssh_tunnel_pid) and (defined $ssh_tunnel_forward_port))
+	{
+		$set_string = "
+ssh_tunnel_host_uuid = ".$anvil->Database->quote($ssh_tunnel_host_uuid).",
+ssh_tunnel_pid = ".$anvil->Database->quote($ssh_tunnel_pid).",
+ssh_tunnel_forward_port = ".$anvil->Database->quote($ssh_tunnel_forward_port)."
+";
+		$condition_string = $condition_string." AND ssh_tunnel_host_uuid = ".$anvil->Database->quote($ssh_tunnel_host_uuid);
+	}
+
+	my $query = "
+UPDATE public.vnc_pipes
+SET ".$set_string."
+WHERE ".$condition_string."
+;";
+
+	$anvil->Database->write({ query => $query, source => $THIS_FILE, line => __LINE__ });
+}
+
 sub get_vnc_pipe
 {
 	my $parameters  = shift;
@@ -611,7 +759,12 @@ sub open_vnc_pipe
 		return;
 	}
 
-	my $ws_info = start_websockify({ server_uuid => $server_uuid, host_name => $server_info->{host_name}, target_port => $vnc_info->{port} });
+	my $ws_info = start_websockify({
+		server_uuid => $server_uuid,
+		host_name   => $server_info->{host_name},
+		host_uuid   => $server_info->{host_uuid},
+		target_port => $vnc_info->{port}
+	});
 
 	if (not defined $ws_info)
 	{
@@ -626,6 +779,7 @@ sub open_vnc_pipe
 		server_uuid    => $server_uuid,
 		host_uuid      => $host_uuid,
 		ws_host_name   => $server_info->{host_name},
+		ws_host_uuid   => $server_info->{host_uuid},
 		ws_source_port => $ws_info->{source_port}
 	});
 
@@ -638,16 +792,17 @@ sub open_vnc_pipe
 		return;
 	}
 
-	if (not $ws_info->{is_existing} or not $ssh_tunnel_info->{is_existing})
+	if ($ws_info->{is_new} and $ssh_tunnel_info->{is_new})
 	{
 		insert_vnc_pipe({
 			server_uuid             => $server_uuid,
 			server_vnc_port         => $vnc_info->{port},
 			ws_host_uuid            => $server_info->{host_uuid},
 			ws_pid                  => $ws_info->{pid},
+			ws_source_port          => $ws_info->{source_port},
 			ssh_tunnel_host_uuid    => $host_uuid,
 			ssh_tunnel_pid          => $ssh_tunnel_info->{pid},
-			ssh_tunnel_forward_port => $ws_info->{source_port}
+			ssh_tunnel_forward_port => $ssh_tunnel_info->{forward_port}
 		});
 	}
 }