Install a "dead man switch" to allow the postmaster to detect cases where

a backend has done exit(0) or exit(1) without having disengaged itself
from shared memory.  We are at risk for this whenever third-party code is
loaded into a backend, since such code might not know it's supposed to go
through proc_exit() instead.  Also, it is reported that under Windows
there are ways to externally kill a process that cause the status code
returned to the postmaster to be indistinguishable from a voluntary exit
(thank you, Microsoft).  If this does happen then the system is probably
hosed --- for instance, the dead session might still be holding locks.
So the best recovery method is to treat this like a backend crash.

The dead man switch is armed for a particular child process when it
acquires a regular PGPROC, and disarmed when the PGPROC is released;
these should be the first and last touches of shared memory resources
in a backend, or close enough anyway.  This choice means there is no
coverage for auxiliary processes, but I doubt we need that, since they
shouldn't be executing any user-provided code anyway.

This patch also improves the management of the EXEC_BACKEND
ShmemBackendArray array a bit, by reducing search costs.

Although this problem is of long standing, the lack of field complaints
seems to mean it's not critical enough to risk back-patching; at least
not till we get some more testing of this mechanism.

Author: tglsfdc

src/backend/postmaster/postmaster.c

@@ -37,7 +37,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.580 2009/05/04 02:46:36 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.581 2009/05/05 19:59:00 tgl Exp $
  *
  * NOTES
  *
@@ -135,12 +135,14 @@
  * Also, "dead_end" children are in it: these are children launched just
  * for the purpose of sending a friendly rejection message to a would-be
  * client.	We must track them because they are attached to shared memory,
- * but we know they will never become live backends.
+ * but we know they will never become live backends.  dead_end children are
+ * not assigned a PMChildSlot.
  */
 typedef struct bkend
 {
 	pid_t		pid;			/* process id of backend */
 	long		cancel_key;		/* cancel key for cancels for this backend */
+	int			child_slot;		/* PMChildSlot for this backend, if any */
 	bool		is_autovacuum;	/* is it an autovacuum process? */
 	bool		dead_end;		/* is it going to send an error and quit? */
 	Dlelem		elem;			/* list link in BackendList */
@@ -149,15 +151,6 @@ typedef struct bkend
 static Dllist *BackendList;
 
 #ifdef EXEC_BACKEND
-/*
- * Number of entries in the shared-memory backend table.  This table is used
- * only for sending cancels, and therefore only includes children we allow
- * cancels on: regular backends and autovac workers.  In particular we exclude
- * dead_end children, allowing the table to have a known maximum size, to wit
- * the same too-many-children limit enforced by canAcceptConnections().
- */
-#define NUM_BACKENDARRAY_ELEMS (2*MaxBackends)
-
 static Backend *ShmemBackendArray;
 #endif
 
@@ -404,6 +397,7 @@ typedef struct
 	char		DataDir[MAXPGPATH];
 	int			ListenSocket[MAXLISTEN];
 	long		MyCancelKey;
+	int			MyPMChildSlot;
 	unsigned long UsedShmemSegID;
 	void	   *UsedShmemSegAddr;
 	slock_t    *ShmemLock;
@@ -413,6 +407,7 @@ typedef struct
 	slock_t    *ProcStructLock;
 	PROC_HDR   *ProcGlobal;
 	PGPROC	   *AuxiliaryProcs;
+	PMSignalData *PMSignalState;
 	InheritableSocket pgStatSock;
 	pid_t		PostmasterPid;
 	TimestampTz PgStartTime;
@@ -443,7 +438,7 @@ static bool save_backend_variables(BackendParameters * param, Port *port,
 #endif
 
 static void ShmemBackendArrayAdd(Backend *bn);
-static void ShmemBackendArrayRemove(pid_t pid);
+static void ShmemBackendArrayRemove(Backend *bn);
 #endif   /* EXEC_BACKEND */
 
 #define StartupDataBase()		StartChildProcess(StartupProcess)
@@ -1771,7 +1766,7 @@ processCancelRequest(Port *port, void *pkt)
 	{
 		bp = (Backend *) DLE_VAL(curr);
 #else
-	for (i = 0; i < NUM_BACKENDARRAY_ELEMS; i++)
+	for (i = MaxLivePostmasterChildren() - 1; i >= 0; i--)
 	{
 		bp = (Backend *) &ShmemBackendArray[i];
 #endif
@@ -1836,10 +1831,10 @@ canAcceptConnections(void)
 	 * MaxBackends limit is enforced when a new backend tries to join the
 	 * shared-inval backend array.
 	 *
-	 * In the EXEC_BACKEND case, the limit here must match the size of the
-	 * ShmemBackendArray, since all these processes will have cancel codes.
+	 * The limit here must match the sizes of the per-child-process arrays;
+	 * see comments for MaxLivePostmasterChildren().
 	 */
-	if (CountChildren() >= 2 * MaxBackends)
+	if (CountChildren() >= MaxLivePostmasterChildren())
 		return CAC_TOOMANY;
 
 	return CAC_OK;
@@ -2439,8 +2434,8 @@ CleanupBackend(int pid,
 	/*
 	 * If a backend dies in an ugly way then we must signal all other backends
 	 * to quickdie.  If exit status is zero (normal) or one (FATAL exit), we
-	 * assume everything is all right and simply remove the backend from the
-	 * active backend list.
+	 * assume everything is all right and proceed to remove the backend from
+	 * the active backend list.
 	 */
 	if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus))
 	{
@@ -2454,10 +2449,21 @@ CleanupBackend(int pid,
 
 		if (bp->pid == pid)
 		{
-#ifdef EXEC_BACKEND
 			if (!bp->dead_end)
-				ShmemBackendArrayRemove(pid);
+			{
+				if (!ReleasePostmasterChildSlot(bp->child_slot))
+				{
+					/*
+					 * Uh-oh, the child failed to clean itself up.  Treat
+					 * as a crash after all.
+					 */
+					HandleChildCrash(pid, exitstatus, _("server process"));
+					return;
+				}
+#ifdef EXEC_BACKEND
+				ShmemBackendArrayRemove(bp);
 #endif
+			}
 			DLRemove(curr);
 			free(bp);
 			break;
@@ -2500,10 +2506,13 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
 			/*
 			 * Found entry for freshly-dead backend, so remove it.
 			 */
-#ifdef EXEC_BACKEND
 			if (!bp->dead_end)
-				ShmemBackendArrayRemove(pid);
+			{
+				(void) ReleasePostmasterChildSlot(bp->child_slot);
+#ifdef EXEC_BACKEND
+				ShmemBackendArrayRemove(bp);
 #endif
+			}
 			DLRemove(curr);
 			free(bp);
 			/* Keep looping so we can signal remaining backends */
@@ -2931,14 +2940,7 @@ BackendStartup(Port *port)
 	pid_t		pid;
 
 	/*
-	 * Compute the cancel key that will be assigned to this backend. The
-	 * backend will have its own copy in the forked-off process' value of
-	 * MyCancelKey, so that it can transmit the key to the frontend.
-	 */
-	MyCancelKey = PostmasterRandom();
-
-	/*
-	 * Make room for backend data structure.  Better before the fork() so we
+	 * Create backend data structure.  Better before the fork() so we
 	 * can handle failure cleanly.
 	 */
 	bn = (Backend *) malloc(sizeof(Backend));
@@ -2950,8 +2952,26 @@ BackendStartup(Port *port)
 		return STATUS_ERROR;
 	}
 
+	/*
+	 * Compute the cancel key that will be assigned to this backend. The
+	 * backend will have its own copy in the forked-off process' value of
+	 * MyCancelKey, so that it can transmit the key to the frontend.
+	 */
+	MyCancelKey = PostmasterRandom();
+	bn->cancel_key = MyCancelKey;
+
 	/* Pass down canAcceptConnections state */
 	port->canAcceptConnections = canAcceptConnections();
+	bn->dead_end = (port->canAcceptConnections != CAC_OK &&
+					port->canAcceptConnections != CAC_WAITBACKUP);
+
+	/*
+	 * Unless it's a dead_end child, assign it a child slot number
+	 */
+	if (!bn->dead_end)
+		bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot();
+	else
+		bn->child_slot = 0;
 
 #ifdef EXEC_BACKEND
 	pid = backend_forkexec(port);
@@ -3009,10 +3029,7 @@ BackendStartup(Port *port)
 	 * of backends.
 	 */
 	bn->pid = pid;
-	bn->cancel_key = MyCancelKey;
 	bn->is_autovacuum = false;
-	bn->dead_end = (port->canAcceptConnections != CAC_OK &&
-					port->canAcceptConnections != CAC_WAITBACKUP);
 	DLInitElem(&bn->elem, bn);
 	DLAddHead(BackendList, &bn->elem);
 #ifdef EXEC_BACKEND
@@ -4271,23 +4288,26 @@ StartAutovacuumWorker(void)
 	 */
 	if (canAcceptConnections() == CAC_OK)
 	{
-		/*
-		 * Compute the cancel key that will be assigned to this session. We
-		 * probably don't need cancel keys for autovac workers, but we'd
-		 * better have something random in the field to prevent unfriendly
-		 * people from sending cancels to them.
-		 */
-		MyCancelKey = PostmasterRandom();
-
 		bn = (Backend *) malloc(sizeof(Backend));
 		if (bn)
 		{
+			/*
+			 * Compute the cancel key that will be assigned to this session. We
+			 * probably don't need cancel keys for autovac workers, but we'd
+			 * better have something random in the field to prevent unfriendly
+			 * people from sending cancels to them.
+			 */
+			MyCancelKey = PostmasterRandom();
+			bn->cancel_key = MyCancelKey;
+
+			/* Autovac workers are not dead_end and need a child slot */
+			bn->dead_end = false;
+			bn->child_slot = MyPMChildSlot = AssignPostmasterChildSlot();
+
 			bn->pid = StartAutoVacWorker();
 			if (bn->pid > 0)
 			{
-				bn->cancel_key = MyCancelKey;
 				bn->is_autovacuum = true;
-				bn->dead_end = false;
 				DLInitElem(&bn->elem, bn);
 				DLAddHead(BackendList, &bn->elem);
 #ifdef EXEC_BACKEND
@@ -4353,6 +4373,24 @@ CreateOptsFile(int argc, char *argv[], char *fullprogname)
 }
 
 
+/*
+ * MaxLivePostmasterChildren
+ *
+ * This reports the number of entries needed in per-child-process arrays
+ * (the PMChildFlags array, and if EXEC_BACKEND the ShmemBackendArray).
+ * These arrays include regular backends and autovac workers, but not special
+ * children nor dead_end children.  This allows the arrays to have a fixed
+ * maximum size, to wit the same too-many-children limit enforced by
+ * canAcceptConnections().  The exact value isn't too critical as long as
+ * it's more than MaxBackends.
+ */
+int
+MaxLivePostmasterChildren(void)
+{
+	return 2 * MaxBackends;
+}
+
+
 #ifdef EXEC_BACKEND
 
 /*
@@ -4364,6 +4402,7 @@ extern LWLock *LWLockArray;
 extern slock_t *ProcStructLock;
 extern PROC_HDR *ProcGlobal;
 extern PGPROC *AuxiliaryProcs;
+extern PMSignalData *PMSignalState;
 extern int	pgStatSock;
 
 #ifndef WIN32
@@ -4395,6 +4434,7 @@ save_backend_variables(BackendParameters * param, Port *port,
 	memcpy(&param->ListenSocket, &ListenSocket, sizeof(ListenSocket));
 
 	param->MyCancelKey = MyCancelKey;
+	param->MyPMChildSlot = MyPMChildSlot;
 
 	param->UsedShmemSegID = UsedShmemSegID;
 	param->UsedShmemSegAddr = UsedShmemSegAddr;
@@ -4407,6 +4447,7 @@ save_backend_variables(BackendParameters * param, Port *port,
 	param->ProcStructLock = ProcStructLock;
 	param->ProcGlobal = ProcGlobal;
 	param->AuxiliaryProcs = AuxiliaryProcs;
+	param->PMSignalState = PMSignalState;
 	write_inheritable_socket(&param->pgStatSock, pgStatSock, childPid);
 
 	param->PostmasterPid = PostmasterPid;
@@ -4601,6 +4642,7 @@ restore_backend_variables(BackendParameters * param, Port *port)
 	memcpy(&ListenSocket, &param->ListenSocket, sizeof(ListenSocket));
 
 	MyCancelKey = param->MyCancelKey;
+	MyPMChildSlot = param->MyPMChildSlot;
 
 	UsedShmemSegID = param->UsedShmemSegID;
 	UsedShmemSegAddr = param->UsedShmemSegAddr;
@@ -4613,6 +4655,7 @@ restore_backend_variables(BackendParameters * param, Port *port)
 	ProcStructLock = param->ProcStructLock;
 	ProcGlobal = param->ProcGlobal;
 	AuxiliaryProcs = param->AuxiliaryProcs;
+	PMSignalState = param->PMSignalState;
 	read_inheritable_socket(&pgStatSock, &param->pgStatSock);
 
 	PostmasterPid = param->PostmasterPid;
@@ -4642,7 +4685,7 @@ restore_backend_variables(BackendParameters * param, Port *port)
 Size
 ShmemBackendArraySize(void)
 {
-	return mul_size(NUM_BACKENDARRAY_ELEMS, sizeof(Backend));
+	return mul_size(MaxLivePostmasterChildren(), sizeof(Backend));
 }
 
 void
@@ -4658,41 +4701,23 @@ ShmemBackendArrayAllocation(void)
 static void
 ShmemBackendArrayAdd(Backend *bn)
 {
-	int			i;
-
-	/* Find an empty slot */
-	for (i = 0; i < NUM_BACKENDARRAY_ELEMS; i++)
-	{
-		if (ShmemBackendArray[i].pid == 0)
-		{
-			ShmemBackendArray[i] = *bn;
-			return;
-		}
-	}
+	/* The array slot corresponding to my PMChildSlot should be free */
+	int			i = bn->child_slot - 1;
 
-	ereport(FATAL,
-			(errmsg_internal("no free slots in shmem backend array")));
+	Assert(ShmemBackendArray[i].pid == 0);
+	ShmemBackendArray[i] = *bn;
 }
 
 static void
-ShmemBackendArrayRemove(pid_t pid)
+ShmemBackendArrayRemove(Backend *bn)
 {
-	int			i;
-
-	for (i = 0; i < NUM_BACKENDARRAY_ELEMS; i++)
-	{
-		if (ShmemBackendArray[i].pid == pid)
-		{
-			/* Mark the slot as empty */
-			ShmemBackendArray[i].pid = 0;
-			return;
-		}
-	}
+	int			i = bn->child_slot - 1;
 
-	ereport(WARNING,
-			(errmsg_internal("could not find backend entry with pid %d",
-							 (int) pid)));
+	Assert(ShmemBackendArray[i].pid == bn->pid);
+	/* Mark the slot as empty */
+	ShmemBackendArray[i].pid = 0;
 }
+
 #endif   /* EXEC_BACKEND */
 
 

src/backend/storage/ipc/ipci.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.99 2009/01/03 17:08:39 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.100 2009/05/05 19:59:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -111,6 +111,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, ProcArrayShmemSize());
 		size = add_size(size, BackendStatusShmemSize());
 		size = add_size(size, SInvalShmemSize());
+		size = add_size(size, PMSignalShmemSize());
 		size = add_size(size, BgWriterShmemSize());
 		size = add_size(size, AutoVacuumShmemSize());
 		size = add_size(size, BTreeShmemSize());
@@ -206,7 +207,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	/*
 	 * Set up interprocess signaling mechanisms
 	 */
-	PMSignalInit();
+	PMSignalShmemInit();
 	BgWriterShmemInit();
 	AutoVacuumShmemInit();