Enhance libpq encryption negotiation tests with new GUC

The new "log_connection_negotiation" server option causes the server
to print messages to the log when it receives a SSLRequest or
GSSENCRequest packet from the client. Together with "log_connections",
it gives a trace of how a connection and encryption is
negotiatated. Use the option in the libpq_encryption test, to verify
in more detail how libpq negotiates encryption with different
gssencmode and sslmode options.

This revealed a couple of cases where libpq retries encryption or
authentication, when it should already know that it cannot succeed.  I
marked them with XXX comments in the test tables. They only happen
when the connection was going to fail anyway, and only with rare
combinations of options, so they're not serious.

Discussion: https://www.postgresql.org/message-id/CAEze2Wja8VUoZygCepwUeiCrWa4jP316k0mvJrOW4PFmWP0Tcw@mail.gmail.com

Author: hlinnaka

src/backend/tcop/backend_startup.c

@@ -37,6 +37,9 @@
 #include "utils/ps_status.h"
 #include "utils/timeout.h"
 
+/* GUCs */
+bool		Trace_connection_negotiation = false;
+
 static void BackendInitialize(ClientSocket *client_sock, CAC_state cac);
 static int	ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done);
 static void SendNegotiateProtocolVersion(List *unrecognized_protocol_options);
@@ -474,6 +477,16 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 		SSLok = 'N';			/* No support for SSL */
 #endif
 
+		if (Trace_connection_negotiation)
+		{
+			if (SSLok == 'S')
+				ereport(LOG,
+						(errmsg("SSLRequest accepted")));
+			else
+				ereport(LOG,
+						(errmsg("SSLRequest rejected")));
+		}
+
 retry1:
 		if (send(port->sock, &SSLok, 1, 0) != 1)
 		{
@@ -519,6 +532,16 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 			GSSok = 'G';
 #endif
 
+		if (Trace_connection_negotiation)
+		{
+			if (GSSok == 'G')
+				ereport(LOG,
+						(errmsg("GSSENCRequest accepted")));
+			else
+				ereport(LOG,
+						(errmsg("GSSENCRequest rejected")));
+		}
+
 		while (send(port->sock, &GSSok, 1, 0) != 1)
 		{
 			if (errno == EINTR)

src/backend/utils/misc/guc_tables.c

@@ -96,6 +96,7 @@
 
 /* XXX these should appear in other modules' header files */
 extern bool Log_disconnections;
+extern bool Trace_connection_negotiation;
 extern int	CommitDelay;
 extern int	CommitSiblings;
 extern char *default_tablespace;
@@ -1224,6 +1225,16 @@ struct config_bool ConfigureNamesBool[] =
 		false,
 		NULL, NULL, NULL
 	},
+	{
+		{"trace_connection_negotiation", PGC_POSTMASTER, DEVELOPER_OPTIONS,
+			gettext_noop("Log details of pre-authentication connection handshake."),
+			NULL,
+			GUC_NOT_IN_SAMPLE
+		},
+		&Trace_connection_negotiation,
+		false,
+		NULL, NULL, NULL
+	},
 	{
 		{"log_disconnections", PGC_SU_BACKEND, LOGGING_WHAT,
 			gettext_noop("Logs end of a session, including duration."),