Once again allow LWLocks to be used within DSM segments.

Prior to commit 7882c3b, it was
possible to use LWLocks within DSM segments, but that commit broke
this use case by switching from a doubly linked list to a circular
linked list.  Switch back, using a new bit of general infrastructure
for maintaining lists of PGPROCs.

Thomas Munro, reviewed by me.

Author: robertmhaas

src/backend/storage/lmgr/lwlock.c

@@ -84,6 +84,7 @@
 #include "storage/ipc.h"
 #include "storage/predicate.h"
 #include "storage/proc.h"
+#include "storage/proclist.h"
 #include "storage/spin.h"
 #include "utils/memutils.h"
 
@@ -717,7 +718,7 @@ LWLockInitialize(LWLock *lock, int tranche_id)
 	pg_atomic_init_u32(&lock->nwaiters, 0);
 #endif
 	lock->tranche = tranche_id;
-	dlist_init(&lock->waiters);
+	proclist_init(&lock->waiters);
 }
 
 /*
@@ -920,25 +921,25 @@ LWLockWakeup(LWLock *lock)
 {
 	bool		new_release_ok;
 	bool		wokeup_somebody = false;
-	dlist_head	wakeup;
-	dlist_mutable_iter iter;
+	proclist_head wakeup;
+	proclist_mutable_iter iter;
 
-	dlist_init(&wakeup);
+	proclist_init(&wakeup);
 
 	new_release_ok = true;
 
 	/* lock wait list while collecting backends to wake up */
 	LWLockWaitListLock(lock);
 
-	dlist_foreach_modify(iter, &lock->waiters)
+	proclist_foreach_modify(iter, &lock->waiters, lwWaitLink)
 	{
-		PGPROC	   *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur);
+		PGPROC	   *waiter = GetPGProcByNumber(iter.cur);
 
 		if (wokeup_somebody && waiter->lwWaitMode == LW_EXCLUSIVE)
 			continue;
 
-		dlist_delete(&waiter->lwWaitLink);
-		dlist_push_tail(&wakeup, &waiter->lwWaitLink);
+		proclist_delete(&lock->waiters, iter.cur, lwWaitLink);
+		proclist_push_tail(&wakeup, iter.cur, lwWaitLink);
 
 		if (waiter->lwWaitMode != LW_WAIT_UNTIL_FREE)
 		{
@@ -963,7 +964,7 @@ LWLockWakeup(LWLock *lock)
 			break;
 	}
 
-	Assert(dlist_is_empty(&wakeup) || pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS);
+	Assert(proclist_is_empty(&wakeup) || pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS);
 
 	/* unset required flags, and release lock, in one fell swoop */
 	{
@@ -982,7 +983,7 @@ LWLockWakeup(LWLock *lock)
 			else
 				desired_state &= ~LW_FLAG_RELEASE_OK;
 
-			if (dlist_is_empty(&wakeup))
+			if (proclist_is_empty(&wakeup))
 				desired_state &= ~LW_FLAG_HAS_WAITERS;
 
 			desired_state &= ~LW_FLAG_LOCKED;	/* release lock */
@@ -994,12 +995,12 @@ LWLockWakeup(LWLock *lock)
 	}
 
 	/* Awaken any waiters I removed from the queue. */
-	dlist_foreach_modify(iter, &wakeup)
+	proclist_foreach_modify(iter, &wakeup, lwWaitLink)
 	{
-		PGPROC	   *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur);
+		PGPROC	   *waiter = GetPGProcByNumber(iter.cur);
 
 		LOG_LWDEBUG("LWLockRelease", lock, "release waiter");
-		dlist_delete(&waiter->lwWaitLink);
+		proclist_delete(&wakeup, iter.cur, lwWaitLink);
 
 		/*
 		 * Guarantee that lwWaiting being unset only becomes visible once the
@@ -1046,9 +1047,9 @@ LWLockQueueSelf(LWLock *lock, LWLockMode mode)
 
 	/* LW_WAIT_UNTIL_FREE waiters are always at the front of the queue */
 	if (mode == LW_WAIT_UNTIL_FREE)
-		dlist_push_head(&lock->waiters, &MyProc->lwWaitLink);
+		proclist_push_head(&lock->waiters, MyProc->pgprocno, lwWaitLink);
 	else
-		dlist_push_tail(&lock->waiters, &MyProc->lwWaitLink);
+		proclist_push_tail(&lock->waiters, MyProc->pgprocno, lwWaitLink);
 
 	/* Can release the mutex now */
 	LWLockWaitListUnlock(lock);
@@ -1070,7 +1071,7 @@ static void
 LWLockDequeueSelf(LWLock *lock)
 {
 	bool		found = false;
-	dlist_mutable_iter iter;
+	proclist_mutable_iter iter;
 
 #ifdef LWLOCK_STATS
 	lwlock_stats *lwstats;
@@ -1086,19 +1087,17 @@ LWLockDequeueSelf(LWLock *lock)
 	 * Can't just remove ourselves from the list, but we need to iterate over
 	 * all entries as somebody else could have unqueued us.
 	 */
-	dlist_foreach_modify(iter, &lock->waiters)
+	proclist_foreach_modify(iter, &lock->waiters, lwWaitLink)
 	{
-		PGPROC	   *proc = dlist_container(PGPROC, lwWaitLink, iter.cur);
-
-		if (proc == MyProc)
+		if (iter.cur == MyProc->pgprocno)
 		{
 			found = true;
-			dlist_delete(&proc->lwWaitLink);
+			proclist_delete(&lock->waiters, iter.cur, lwWaitLink);
 			break;
 		}
 	}
 
-	if (dlist_is_empty(&lock->waiters) &&
+	if (proclist_is_empty(&lock->waiters) &&
 		(pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS) != 0)
 	{
 		pg_atomic_fetch_and_u32(&lock->state, ~LW_FLAG_HAS_WAITERS);
@@ -1719,12 +1718,12 @@ LWLockWaitForVar(LWLock *lock, uint64 *valptr, uint64 oldval, uint64 *newval)
 void
 LWLockUpdateVar(LWLock *lock, uint64 *valptr, uint64 val)
 {
-	dlist_head	wakeup;
-	dlist_mutable_iter iter;
+	proclist_head wakeup;
+	proclist_mutable_iter iter;
 
 	PRINT_LWDEBUG("LWLockUpdateVar", lock, LW_EXCLUSIVE);
 
-	dlist_init(&wakeup);
+	proclist_init(&wakeup);
 
 	LWLockWaitListLock(lock);
 
@@ -1737,15 +1736,15 @@ LWLockUpdateVar(LWLock *lock, uint64 *valptr, uint64 val)
 	 * See if there are any LW_WAIT_UNTIL_FREE waiters that need to be woken
 	 * up. They are always in the front of the queue.
 	 */
-	dlist_foreach_modify(iter, &lock->waiters)
+	proclist_foreach_modify(iter, &lock->waiters, lwWaitLink)
 	{
-		PGPROC	   *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur);
+		PGPROC	   *waiter = GetPGProcByNumber(iter.cur);
 
 		if (waiter->lwWaitMode != LW_WAIT_UNTIL_FREE)
 			break;
 
-		dlist_delete(&waiter->lwWaitLink);
-		dlist_push_tail(&wakeup, &waiter->lwWaitLink);
+		proclist_delete(&lock->waiters, iter.cur, lwWaitLink);
+		proclist_push_tail(&wakeup, iter.cur, lwWaitLink);
 	}
 
 	/* We are done updating shared state of the lock itself. */
@@ -1754,11 +1753,11 @@ LWLockUpdateVar(LWLock *lock, uint64 *valptr, uint64 val)
 	/*
 	 * Awaken any waiters I removed from the queue.
 	 */
-	dlist_foreach_modify(iter, &wakeup)
+	proclist_foreach_modify(iter, &wakeup, lwWaitLink)
 	{
-		PGPROC	   *waiter = dlist_container(PGPROC, lwWaitLink, iter.cur);
+		PGPROC	   *waiter = GetPGProcByNumber(iter.cur);
 
-		dlist_delete(&waiter->lwWaitLink);
+		proclist_delete(&wakeup, iter.cur, lwWaitLink);
 		/* check comment in LWLockWakeup() about this barrier */
 		pg_write_barrier();
 		waiter->lwWaiting = false;

src/include/storage/lwlock.h

@@ -18,7 +18,7 @@
 #error "lwlock.h may not be included from frontend code"
 #endif
 
-#include "lib/ilist.h"
+#include "storage/proclist_types.h"
 #include "storage/s_lock.h"
 #include "port/atomics.h"
 
@@ -59,7 +59,7 @@ typedef struct LWLock
 {
 	uint16		tranche;		/* tranche ID */
 	pg_atomic_uint32 state;		/* state of exclusive/nonexclusive lockers */
-	dlist_head	waiters;		/* list of waiting PGPROCs */
+	proclist_head waiters;		/* list of waiting PGPROCs */
 #ifdef LOCK_DEBUG
 	pg_atomic_uint32 nwaiters;	/* number of waiters */
 	struct PGPROC *owner;		/* last exclusive owner of the lock */

src/include/storage/proc.h

@@ -19,6 +19,7 @@
 #include "storage/latch.h"
 #include "storage/lock.h"
 #include "storage/pg_sema.h"
+#include "storage/proclist_types.h"
 
 /*
  * Each backend advertises up to PGPROC_MAX_CACHED_SUBXIDS TransactionIds
@@ -112,7 +113,7 @@ struct PGPROC
 	/* Info about LWLock the process is currently waiting for, if any. */
 	bool		lwWaiting;		/* true if waiting for an LW lock */
 	uint8		lwWaitMode;		/* lwlock mode being waited for */
-	dlist_node	lwWaitLink;		/* position in LW lock wait list */
+	proclist_node lwWaitLink;	/* position in LW lock wait list */
 
 	/* Info about lock the process is currently waiting for, if any. */
 	/* waitLock and waitProcLock are NULL if not currently waiting. */
@@ -243,6 +244,9 @@ extern PROC_HDR *ProcGlobal;
 
 extern PGPROC *PreparedXactProcs;
 
+/* Accessor for PGPROC given a pgprocno. */
+#define GetPGProcByNumber(n) (&ProcGlobal->allProcs[(n)])
+
 /*
  * We set aside some extra PGPROC structures for auxiliary processes,
  * ie things that aren't full-fledged backends but need shmem access.

src/include/storage/proclist.h

@@ -0,0 +1,154 @@
+/*-------------------------------------------------------------------------
+ *
+ * proclist.h
+ *		operations on doubly-linked lists of pgprocnos
+ *
+ * The interface is similar to dlist from ilist.h, but uses pgprocno instead
+ * of pointers.  This allows proclist_head to be mapped at different addresses
+ * in different backends.
+ *
+ * See proclist_types.h for the structs that these functions operate on.  They
+ * are separated to break a header dependency cycle with proc.h.
+ *
+ * Portions Copyright (c) 2016, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		src/include/storage/proclist.h
+ *-------------------------------------------------------------------------
+ */
+#ifndef PROCLIST_H
+#define PROCLIST_H
+
+#include "storage/proc.h"
+#include "storage/proclist_types.h"
+
+/*
+ * Initialize a proclist.
+ */
+static inline void
+proclist_init(proclist_head *list)
+{
+	list->head = list->tail = INVALID_PGPROCNO;
+}
+
+/*
+ * Is the list empty?
+ */
+static inline bool
+proclist_is_empty(proclist_head *list)
+{
+	return list->head == INVALID_PGPROCNO;
+}
+
+/*
+ * Get a pointer to a proclist_node inside a given PGPROC, given a procno and
+ * an offset.
+ */
+static inline proclist_node *
+proclist_node_get(int procno, size_t node_offset)
+{
+	char	   *entry = (char *) GetPGProcByNumber(procno);
+
+	return (proclist_node *) (entry + node_offset);
+}
+
+/*
+ * Insert a node at the beginning of a list.
+ */
+static inline void
+proclist_push_head_offset(proclist_head *list, int procno, size_t node_offset)
+{
+	proclist_node *node = proclist_node_get(procno, node_offset);
+
+	if (list->head == INVALID_PGPROCNO)
+	{
+		Assert(list->tail == INVALID_PGPROCNO);
+		node->next = node->prev = INVALID_PGPROCNO;
+		list->head = list->tail = procno;
+	}
+	else
+	{
+		Assert(list->tail != INVALID_PGPROCNO);
+		node->next = list->head;
+		proclist_node_get(node->next, node_offset)->prev = procno;
+		node->prev = INVALID_PGPROCNO;
+		list->head = procno;
+	}
+}
+
+/*
+ * Insert a node a the end of a list.
+ */
+static inline void
+proclist_push_tail_offset(proclist_head *list, int procno, size_t node_offset)
+{
+	proclist_node *node = proclist_node_get(procno, node_offset);
+
+	if (list->tail == INVALID_PGPROCNO)
+	{
+		Assert(list->head == INVALID_PGPROCNO);
+		node->next = node->prev = INVALID_PGPROCNO;
+		list->head = list->tail = procno;
+	}
+	else
+	{
+		Assert(list->head != INVALID_PGPROCNO);
+		node->prev = list->tail;
+		proclist_node_get(node->prev, node_offset)->next = procno;
+		node->next = INVALID_PGPROCNO;
+		list->tail = procno;
+	}
+}
+
+/*
+ * Delete a node.  The node must be in the list.
+ */
+static inline void
+proclist_delete_offset(proclist_head *list, int procno, size_t node_offset)
+{
+	proclist_node *node = proclist_node_get(procno, node_offset);
+
+	if (node->prev == INVALID_PGPROCNO)
+		list->head = node->next;
+	else
+		proclist_node_get(node->prev, node_offset)->next = node->next;
+
+	if (node->next == INVALID_PGPROCNO)
+		list->tail = node->prev;
+	else
+		proclist_node_get(node->next, node_offset)->prev = node->prev;
+}
+
+/*
+ * Helper macros to avoid repetition of offsetof(PGPROC, <member>).
+ * 'link_member' is the name of a proclist_node member in PGPROC.
+ */
+#define proclist_delete(list, procno, link_member) \
+	proclist_delete_offset((list), (procno), offsetof(PGPROC, link_member))
+#define proclist_push_head(list, procno, link_member) \
+	proclist_push_head_offset((list), (procno), offsetof(PGPROC, link_member))
+#define proclist_push_tail(list, procno, link_member) \
+	proclist_push_tail_offset((list), (procno), offsetof(PGPROC, link_member))
+
+/*
+ * Iterate through the list pointed at by 'lhead', storing the current
+ * position in 'iter'.  'link_member' is the name of a proclist_node member in
+ * PGPROC.  Access the current position with iter.cur.
+ *
+ * The only list modification allowed while iterating is deleting the current
+ * node with proclist_delete(list, iter.cur, node_offset).
+ */
+#define proclist_foreach_modify(iter, lhead, link_member)					\
+	for (AssertVariableIsOfTypeMacro(iter, proclist_mutable_iter),			\
+		 AssertVariableIsOfTypeMacro(lhead, proclist_head *),				\
+		 (iter).cur = (lhead)->head,										\
+		 (iter).next = (iter).cur == INVALID_PGPROCNO ? INVALID_PGPROCNO :	\
+			 proclist_node_get((iter).cur,									\
+							   offsetof(PGPROC, link_member))->next;		\
+		 (iter).cur != INVALID_PGPROCNO;									\
+		 (iter).cur = (iter).next,											\
+		 (iter).next = (iter).cur == INVALID_PGPROCNO ? INVALID_PGPROCNO :	\
+			 proclist_node_get((iter).cur,									\
+							   offsetof(PGPROC, link_member))->next)
+
+#endif