Fix memory leak on DSM slot exhaustion.

If we attempt to create a DSM segment when no slots are available,
we should return the memory to the operating system.  Previously
we did that if the DSM_CREATE_NULL_IF_MAXSEGMENTS flag was
passed in, but we didn't do it if an error was raised.  Repair.

Back-patch to 9.4, where DSM segments arrived.

Author: Thomas Munro
Reviewed-by: Robert Haas
Reported-by: Julian Backes
Discussion: https://postgr.es/m/CA%2BhUKGKAAoEw-R4om0d2YM4eqT1eGEi6%3DQot-3ceDR-SLiWVDw%40mail.gmail.com

Author: macdice

src/backend/storage/ipc/dsm.c

@@ -479,17 +479,16 @@ dsm_create(Size size, int flags)
 	/* Verify that we can support an additional mapping. */
 	if (nitems >= dsm_control->maxitems)
 	{
+		LWLockRelease(DynamicSharedMemoryControlLock);
+		dsm_impl_op(DSM_OP_DESTROY, seg->handle, 0, &seg->impl_private,
+					&seg->mapped_address, &seg->mapped_size, WARNING);
+		if (seg->resowner != NULL)
+			ResourceOwnerForgetDSM(seg->resowner, seg);
+		dlist_delete(&seg->node);
+		pfree(seg);
+
 		if ((flags & DSM_CREATE_NULL_IF_MAXSEGMENTS) != 0)
-		{
-			LWLockRelease(DynamicSharedMemoryControlLock);
-			dsm_impl_op(DSM_OP_DESTROY, seg->handle, 0, &seg->impl_private,
-						&seg->mapped_address, &seg->mapped_size, WARNING);
-			if (seg->resowner != NULL)
-				ResourceOwnerForgetDSM(seg->resowner, seg);
-			dlist_delete(&seg->node);
-			pfree(seg);
 			return NULL;
-		}
 		ereport(ERROR,
 				(errcode(ERRCODE_INSUFFICIENT_RESOURCES),
 				 errmsg("too many dynamic shared memory segments")));