Add test for dead-end backends

The code path for launching a dead-end backend because we're out of
slots was not covered by any tests, so add one. (Some tests did hit
the case of launching a dead-end backend because the server is still
starting up, though, so the gap in our test coverage wasn't as big as
it sounds.)

Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://www.postgresql.org/message-id/a102f15f-eac4-4ff2-af02-f9ff209ec66f@iki.fi

Author: hlinnaka

src/test/perl/PostgreSQL/Test/Cluster.pm

@@ -104,6 +104,7 @@ use File::Path qw(rmtree mkpath);
 use File::Spec;
 use File::stat qw(stat);
 use File::Temp ();
+use IO::Socket::INET;
 use IPC::Run;
 use PostgreSQL::Version;
 use PostgreSQL::Test::RecursiveCopy;
@@ -291,6 +292,83 @@ sub connstr
 
 =pod
 
+=item $node->raw_connect()
+
+Open a raw TCP or Unix domain socket connection to the server. This is
+used by low-level protocol and connection limit tests.
+
+=cut
+
+sub raw_connect
+{
+	my ($self) = @_;
+	my $pgport = $self->port;
+	my $pghost = $self->host;
+
+	my $socket;
+	if ($PostgreSQL::Test::Utils::use_unix_sockets)
+	{
+		require IO::Socket::UNIX;
+		my $path = "$pghost/.s.PGSQL.$pgport";
+
+		$socket = IO::Socket::UNIX->new(
+			Type => SOCK_STREAM(),
+			Peer => $path,
+		) or die "Cannot create socket - $IO::Socket::errstr\n";
+	}
+	else
+	{
+		$socket = IO::Socket::INET->new(
+			PeerHost => $pghost,
+			PeerPort => $pgport,
+			Proto => 'tcp'
+		) or die "Cannot create socket - $IO::Socket::errstr\n";
+	}
+	return $socket;
+}
+
+=pod
+
+=item $node->raw_connect_works()
+
+Check if raw_connect() function works on this platform. This should
+be called to SKIP any tests that require raw_connect().
+
+This tries to connect to the server, to test whether it works or not,,
+so the server is up and running. Otherwise this can return 0 even if
+there's nothing wrong with raw_connect() itself.
+
+Notably, raw_connect() does not work on Unix domain sockets on
+Strawberry perl 5.26.3.1 on Windows, which we use in Cirrus CI images
+as of this writing. It dies with "not implemented on this
+architecture".
+
+=cut
+
+sub raw_connect_works
+{
+	my ($self) = @_;
+
+	# If we're using Unix domain sockets, we need a working
+	# IO::Socket::UNIX implementation.
+	if ($PostgreSQL::Test::Utils::use_unix_sockets)
+	{
+		diag "checking if IO::Socket::UNIX works";
+		eval {
+			my $sock = $self->raw_connect();
+			$sock->close();
+		};
+		if ($@ =~ /not implemented/)
+		{
+			diag "IO::Socket::UNIX does not work: $@";
+			return 0;
+		}
+	}
+	return 1
+}
+
+=pod
+
 =item $node->group_access()
 
 Does the data dir allow group access?

src/test/postmaster/t/001_connection_limits.pl

@@ -43,6 +43,7 @@ sub background_psql_as_user
 }
 
 my @sessions = ();
+my @raw_connections = ();
 
 push(@sessions, background_psql_as_user('regress_regular'));
 push(@sessions, background_psql_as_user('regress_regular'));
@@ -69,11 +70,50 @@ sub background_psql_as_user
 	"superuser_reserved_connections limit",
 	expected_stderr => qr/FATAL:  sorry, too many clients already/);
 
-# TODO: test that query cancellation is still possible
+# We can still open TCP (or Unix domain socket) connections, but
+# beyond a certain number (roughly 2x max_connections), they will be
+# "dead-end backends".
+SKIP:
+{
+	skip "this test requies working raw_connect()" unless $node->raw_connect_works();
+
+	for (my $i = 0; $i <= 20; $i++)
+	{
+		my $sock = $node->raw_connect();
+
+		# On a busy system, the server might reject connections if
+		# postmaster cannot accept() them fast enough. The exact limit
+		# and behavior depends on the platform. To make this reliable,
+		# we attempt SSL negotiation on each connection before opening
+		# next one. The server will reject the SSL negotations, but
+		# when it does so, we know that the backend has been launched
+		# and we should be able to open another connection.
+
+		# SSLRequest packet consists of packet length followed by
+		# NEGOTIATE_SSL_CODE.
+		my $negotiate_ssl_code = pack("Nnn", 8, 1234, 5679);
+		my $sent = $sock->send($negotiate_ssl_code);
+
+		# Read reply. We expect the server to reject it with 'N'
+		my $reply = "";
+		$sock->recv($reply, 1);
+		is($reply, "N", "dead-end connection $i");
 
+		push(@raw_connections, $sock);
+	}
+}
+
+# TODO: test that query cancellation is still possible. A dead-end
+# backend can process a query cancellation packet.
+
+# Clean up
 foreach my $session (@sessions)
 {
 	$session->quit;
 }
+foreach my $socket (@raw_connections)
+{
+	$socket->close();
+}
 
 done_testing();