Revert pg_wal_replay_wait() stored procedure

This commit reverts 3c5db1d, and subsequent improvements and fixes
including 8036d73, 867d396, 3ac3ec5, 0868d7a, 85b98b8,
2520226, 014f9f3, e658038, e155564, 5035172, 6cfebfe,
73da6b8, and e546989.

The reason for reverting is a set of remaining issues.  Most notably, the
stored procedure appears to need more effort than the utility statement
to turn the backend into a "snapshot-less" state.  This makes an approach
to use stored procedures questionable.

Catversion is bumped.

Discussion: https://postgr.es/m/Zyhj2anOPRKtb0xW%40paquier.xyz

Author: akorotkov

doc/src/sgml/func.sgml

@@ -29000,176 +29000,6 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
     the pause, the rate of WAL generation and available disk space.
    </para>
 
-   <para>
-    The procedure shown in <xref linkend="recovery-synchronization-procedure-table"/>
-    can be executed only during recovery.
-   </para>
-
-   <table id="recovery-synchronization-procedure-table">
-    <title>Recovery Synchronization Procedure and Function</title>
-    <tgroup cols="1">
-     <thead>
-      <row>
-       <entry role="func_table_entry"><para role="func_signature">
-        Procedure or Function
-       </para>
-       <para>
-        Type
-       </para>
-       <para>
-        Description
-       </para></entry>
-      </row>
-     </thead>
-
-     <tbody>
-      <row>
-       <entry role="func_table_entry"><para role="func_signature">
-        <indexterm>
-         <primary>pg_wal_replay_wait</primary>
-        </indexterm>
-        <function>pg_wal_replay_wait</function> (
-          <parameter>target_lsn</parameter> <type>pg_lsn</type>,
-          <parameter>timeout</parameter> <type>bigint</type> <literal>DEFAULT</literal> <literal>0</literal>,
-          <parameter>no_error</parameter> <type>bool</type> <literal>DEFAULT</literal> <literal>false</literal>)
-       </para>
-       <para>
-        Procedure
-       </para>
-       <para>
-        Waits until recovery replays <literal>target_lsn</literal>.
-        If no <parameter>timeout</parameter> is specified or it is set to
-        zero, this procedure waits indefinitely for the
-        <literal>target_lsn</literal>.  If the <parameter>timeout</parameter>
-        is specified (in milliseconds) and is greater than zero, the
-        procedure waits until <literal>target_lsn</literal> is reached or
-        the specified <parameter>timeout</parameter> has elapsed.
-        On timeout, or if the server is promoted before
-        <literal>target_lsn</literal> is reached, an error is emitted,
-        as soon as <parameter>no_error</parameter> is false.
-        If <parameter>no_error</parameter> is set to true, then the procedure
-        doesn't throw errors.  The last result status could be read
-        with <function>pg_wal_replay_wait_status</function>.
-       </para></entry>
-      </row>
-
-      <row>
-       <entry role="func_table_entry"><para role="func_signature">
-        <indexterm>
-         <primary>pg_wal_replay_wait_status</primary>
-        </indexterm>
-        <function>pg_wal_replay_wait_status</function> ()
-        <returnvalue>text</returnvalue>
-       </para>
-       <para>
-        Function
-       </para>
-       <para>
-        Returns the last result status for
-        <function>pg_wal_replay_wait</function> procedure.  The possible
-        values are <literal>success</literal>, <literal>timeout</literal>,
-        and <literal>not in recovery</literal>.
-       </para></entry>
-      </row>
-     </tbody>
-    </tgroup>
-   </table>
-
-   <para>
-    <function>pg_wal_replay_wait</function> waits till
-    <parameter>target_lsn</parameter> to be replayed on standby.
-    That is, after this function execution, the value returned by
-    <function>pg_last_wal_replay_lsn</function> should be greater or equal
-    to the <parameter>target_lsn</parameter> value.  This is useful to achieve
-    read-your-writes-consistency, while using async replica for reads and
-    primary for writes.  In that case <acronym>lsn</acronym> of the last
-    modification should be stored on the client application side or the
-    connection pooler side.
-   </para>
-
-   <para>
-    <function>pg_wal_replay_wait</function> should be called on standby.
-    If a user calls <function>pg_wal_replay_wait</function> on primary, it
-    will error out as soon as <parameter>no_error</parameter> is false.
-    However, if <function>pg_wal_replay_wait</function> is
-    called on primary promoted from standby and <literal>target_lsn</literal>
-    was already replayed, then <function>pg_wal_replay_wait</function> just
-    exits immediately.
-   </para>
-
-   <para>
-    You can use <function>pg_wal_replay_wait</function> to wait for
-    the <type>pg_lsn</type> value.  For example, an application could update
-    the <literal>movie</literal> table and get the <acronym>lsn</acronym> after
-    changes just made.  This example uses <function>pg_current_wal_insert_lsn</function>
-    on primary server to get the <acronym>lsn</acronym> given that
-    <varname>synchronous_commit</varname> could be set to
-    <literal>off</literal>.
-
-   <programlisting>
-postgres=# UPDATE movie SET genre = 'Dramatic' WHERE genre = 'Drama';
-UPDATE 100
-postgres=# SELECT pg_current_wal_insert_lsn();
-pg_current_wal_insert_lsn
---------------------
-0/306EE20
-(1 row)
-   </programlisting>
-
-   Then an application could run <function>pg_wal_replay_wait</function>
-   with the <acronym>lsn</acronym> obtained from primary.  After that the
-   changes made on primary should be guaranteed to be visible on replica.
-
-   <programlisting>
-postgres=# CALL pg_wal_replay_wait('0/306EE20');
-CALL
-postgres=# SELECT * FROM movie WHERE genre = 'Drama';
- genre
--------
-(0 rows)
-   </programlisting>
-
-   It may also happen that target <acronym>lsn</acronym> is not reached
-   within the timeout.  In that case the error is thrown.
-
-   <programlisting>
-postgres=# CALL pg_wal_replay_wait('0/306EE20', 100);
-ERROR:  timed out while waiting for target LSN 0/306EE20 to be replayed; current replay LSN 0/306EA60
-    </programlisting>
-
-   The same example uses <function>pg_wal_replay_wait</function> with
-   <parameter>no_error</parameter> set to true.  In this case, the result
-   status must be read with <function>pg_wal_replay_wait_status</function>.
-
-   <programlisting>
-postgres=# CALL pg_wal_replay_wait('0/306EE20', 100, true);
-CALL
-postgres=# SELECT pg_wal_replay_wait_status();
- pg_wal_replay_wait_status
----------------------------
- timeout
-(1 row)
-    </programlisting>
-
-   </para>
-
-   <para>
-     <function>pg_wal_replay_wait</function> can't be used within
-     a transaction with an isolation level higher than
-     <literal>READ COMMITTED</literal>, another procedure, or a function.
-     All the cases above imply holding a snapshot, which could prevent
-     WAL records from replaying (see <xref linkend="hot-standby-conflict"/>)
-     and cause an indirect deadlock.
-
-   <programlisting>
-postgres=# BEGIN;
-BEGIN
-postgres=*# CALL pg_wal_replay_wait('0/306EE20');
-ERROR:  pg_wal_replay_wait() must be only called without an active or registered snapshot
-DETAIL:  Make sure pg_wal_replay_wait() isn't called within a transaction with an isolation level higher than READ COMMITTED, another procedure, or a function.
-   </programlisting>
-
-   </para>
   </sect2>
 
   <sect2 id="functions-snapshot-synchronization">

src/backend/access/transam/Makefile

@@ -36,8 +36,7 @@ OBJS = \
 	xlogreader.o \
 	xlogrecovery.o \
 	xlogstats.o \
-	xlogutils.o \
-	xlogwait.o
+	xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
 

src/backend/access/transam/meson.build

@@ -24,7 +24,6 @@ backend_sources += files(
   'xlogrecovery.c',
   'xlogstats.c',
   'xlogutils.c',
-  'xlogwait.c',
 )
 
 # used by frontend programs to build a frontend xlogreader

src/backend/access/transam/xact.c

@@ -31,7 +31,6 @@
 #include "access/xloginsert.h"
 #include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
-#include "access/xlogwait.h"
 #include "catalog/index.h"
 #include "catalog/namespace.h"
 #include "catalog/pg_enum.h"
@@ -2827,11 +2826,6 @@ AbortTransaction(void)
 	 */
 	LWLockReleaseAll();
 
-	/*
-	 * Cleanup waiting for LSN if any.
-	 */
-	WaitLSNCleanup();
-
 	/* Clear wait information and command progress indicator */
 	pgstat_report_wait_end();
 	pgstat_progress_end_command();

src/backend/access/transam/xlog.c

@@ -62,7 +62,6 @@
 #include "access/xlogreader.h"
 #include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
-#include "access/xlogwait.h"
 #include "backup/basebackup.h"
 #include "catalog/catversion.h"
 #include "catalog/pg_control.h"
@@ -6174,12 +6173,6 @@ StartupXLOG(void)
 	UpdateControlFile();
 	LWLockRelease(ControlFileLock);
 
-	/*
-	 * Wake up all waiters for replay LSN.  They need to report an error that
-	 * recovery was ended before reaching the target LSN.
-	 */
-	WaitLSNWakeup(InvalidXLogRecPtr);
-
 	/*
 	 * Shutdown the recovery environment.  This must occur after
 	 * RecoverPreparedTransactions() (see notes in lock_twophase_recover())

src/backend/access/transam/xlogfuncs.c

@@ -22,19 +22,17 @@
 #include "access/xlog_internal.h"
 #include "access/xlogbackup.h"
 #include "access/xlogrecovery.h"
-#include "access/xlogwait.h"
 #include "catalog/pg_type.h"
 #include "funcapi.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "replication/walreceiver.h"
 #include "storage/fd.h"
-#include "storage/proc.h"
+#include "storage/latch.h"
 #include "storage/standby.h"
 #include "utils/builtins.h"
 #include "utils/memutils.h"
 #include "utils/pg_lsn.h"
-#include "utils/snapmgr.h"
 #include "utils/timestamp.h"
 
 /*
@@ -750,115 +748,3 @@ pg_promote(PG_FUNCTION_ARGS)
 						   wait_seconds)));
 	PG_RETURN_BOOL(false);
 }
-
-static WaitLSNResult lastWaitLSNResult = WAIT_LSN_RESULT_SUCCESS;
-
-/*
- * Waits until recovery replays the target LSN with optional timeout.  Unless
- * 'no_error' provided throws an error on failure
- */
-Datum
-pg_wal_replay_wait(PG_FUNCTION_ARGS)
-{
-	XLogRecPtr	target_lsn = PG_GETARG_LSN(0);
-	int64		timeout = PG_GETARG_INT64(1);
-	bool		no_error = PG_GETARG_BOOL(2);
-
-	if (timeout < 0)
-		ereport(ERROR,
-				(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
-				 errmsg("\"timeout\" must not be negative")));
-
-	/*
-	 * We are going to wait for the LSN replay.  We should first care that we
-	 * don't hold a snapshot and correspondingly our MyProc->xmin is invalid.
-	 * Otherwise, our snapshot could prevent the replay of WAL records
-	 * implying a kind of self-deadlock.  This is the reason why
-	 * pg_wal_replay_wait() is a procedure, not a function.
-	 *
-	 * At first, we should check there is no active snapshot.  According to
-	 * PlannedStmtRequiresSnapshot(), even in an atomic context, CallStmt is
-	 * processed with a snapshot.  Thankfully, we can pop this snapshot,
-	 * because PortalRunUtility() can tolerate this.
-	 */
-	if (ActiveSnapshotSet())
-		PopActiveSnapshot();
-
-	/*
-	 * At second, invalidate a catalog snapshot if any.  And we should be done
-	 * with the preparation.
-	 */
-	InvalidateCatalogSnapshot();
-
-	/* Give up if there is still an active or registered snapshot. */
-	if (GetOldestSnapshot())
-		ereport(ERROR,
-				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-				 errmsg("pg_wal_replay_wait() must be only called without an active or registered snapshot"),
-				 errdetail("Make sure pg_wal_replay_wait() isn't called within a transaction with an isolation level higher than READ COMMITTED, another procedure, or a function.")));
-
-	/*
-	 * As the result we should hold no snapshot, and correspondingly our xmin
-	 * should be unset.
-	 */
-	Assert(MyProc->xmin == InvalidTransactionId);
-
-	lastWaitLSNResult = WaitForLSNReplay(target_lsn, timeout);
-
-	if (no_error)
-		PG_RETURN_VOID();
-
-	/*
-	 * Process the result of WaitForLSNReplay().  Throw appropriate error if
-	 * needed.
-	 */
-	switch (lastWaitLSNResult)
-	{
-		case WAIT_LSN_RESULT_SUCCESS:
-			/* Nothing to do on success */
-			break;
-
-		case WAIT_LSN_RESULT_TIMEOUT:
-			ereport(ERROR,
-					(errcode(ERRCODE_QUERY_CANCELED),
-					 errmsg("timed out while waiting for target LSN %X/%X to be replayed; current replay LSN %X/%X",
-							LSN_FORMAT_ARGS(target_lsn),
-							LSN_FORMAT_ARGS(GetXLogReplayRecPtr(NULL)))));
-			break;
-
-		case WAIT_LSN_RESULT_NOT_IN_RECOVERY:
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("recovery is not in progress"),
-					 errdetail("Recovery ended before replaying target LSN %X/%X; last replay LSN %X/%X.",
-							   LSN_FORMAT_ARGS(target_lsn),
-							   LSN_FORMAT_ARGS(GetXLogReplayRecPtr(NULL)))));
-			break;
-	}
-
-	PG_RETURN_VOID();
-}
-
-Datum
-pg_wal_replay_wait_status(PG_FUNCTION_ARGS)
-{
-	const char *result_string = "";
-
-	/* Process the result of WaitForLSNReplay(). */
-	switch (lastWaitLSNResult)
-	{
-		case WAIT_LSN_RESULT_SUCCESS:
-			result_string = "success";
-			break;
-
-		case WAIT_LSN_RESULT_TIMEOUT:
-			result_string = "timeout";
-			break;
-
-		case WAIT_LSN_RESULT_NOT_IN_RECOVERY:
-			result_string = "not in recovery";
-			break;
-	}
-
-	PG_RETURN_TEXT_P(cstring_to_text(result_string));
-}

src/backend/access/transam/xlogrecovery.c

@@ -40,7 +40,6 @@
 #include "access/xlogreader.h"
 #include "access/xlogrecovery.h"
 #include "access/xlogutils.h"
-#include "access/xlogwait.h"
 #include "backup/basebackup.h"
 #include "catalog/pg_control.h"
 #include "commands/tablespace.h"
@@ -1829,16 +1828,6 @@ PerformWalRecovery(void)
 				break;
 			}
 
-			/*
-			 * If we replayed an LSN that someone was waiting for then walk
-			 * over the shared memory array and set latches to notify the
-			 * waiters.
-			 */
-			if (waitLSNState &&
-				(XLogRecoveryCtl->lastReplayedEndRecPtr >=
-				 pg_atomic_read_u64(&waitLSNState->minWaitedLSN)))
-				WaitLSNWakeup(XLogRecoveryCtl->lastReplayedEndRecPtr);
-
 			/* Else, try to fetch the next WAL record */
 			record = ReadRecord(xlogprefetcher, LOG, false, replayTLI);
 		} while (record != NULL);