pg_basebackup: Add --nosync option

This is useful for testing, similar to initdb's --nosync.

From: Michael Paquier <michael.paquier@gmail.com>

Author: petere

doc/src/sgml/ref/pg_basebackup.sgml

@@ -438,6 +438,21 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-N</option></term>
+      <term><option>--nosync</option></term>
+      <listitem>
+       <para>
+        By default, <command>pg_basebackup</command> will wait for all files
+        to be written safely to disk.  This option causes
+        <command>pg_basebackup</command> to return without waiting, which is
+        faster, but means that a subsequent operating system crash can leave
+        the base backup corrupt.  Generally, this option is useful for testing
+        but should not be used when creating a production installation.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-v</option></term>
       <term><option>--verbose</option></term>

src/bin/pg_basebackup/pg_basebackup.c

@@ -69,6 +69,7 @@ static bool includewal = false;
 static bool streamwal = false;
 static bool fastcheckpoint = false;
 static bool writerecoveryconf = false;
+static bool do_sync = true;
 static int	standby_message_timeout = 10 * 1000;		/* 10 sec = default */
 static pg_time_t last_progress_report = 0;
 static int32 maxrate = 0;		/* no limit by default */
@@ -329,6 +330,7 @@ usage(void)
 			 "                         set fast or spread checkpointing\n"));
 	printf(_("  -l, --label=LABEL      set backup label\n"));
 	printf(_("  -n, --noclean          do not clean up after errors\n"));
+	printf(_("  -N, --nosync           do not wait for changes to be written safely to disk\n"));
 	printf(_("  -P, --progress         show progress information\n"));
 	printf(_("  -v, --verbose          output verbose messages\n"));
 	printf(_("  -V, --version          output version information, then exit\n"));
@@ -460,6 +462,7 @@ LogStreamerMain(logstreamer_param *param)
 	stream.stream_stop = reached_end_position;
 	stream.standby_message_timeout = standby_message_timeout;
 	stream.synchronous = false;
+	stream.do_sync = do_sync;
 	stream.mark_done = true;
 	stream.basedir = param->xlogdir;
 	stream.partial_suffix = NULL;
@@ -1199,7 +1202,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
 		PQfreemem(copybuf);
 
 	/* sync the resulting tar file, errors are not considered fatal */
-	if (strcmp(basedir, "-") != 0)
+	if (do_sync && strcmp(basedir, "-") != 0)
 		(void) fsync_fname(filename, false, progname);
 }
 
@@ -1967,14 +1970,17 @@ BaseBackup(void)
 	 * all the data of the base directory is synced, taking into account
 	 * all the tablespaces. Errors are not considered fatal.
 	 */
-	if (format == 't')
+	if (do_sync)
 	{
-		if (strcmp(basedir, "-") != 0)
-			(void) fsync_fname(basedir, true, progname);
-	}
-	else
-	{
-		(void) fsync_pgdata(basedir, progname);
+		if (format == 't')
+		{
+			if (strcmp(basedir, "-") != 0)
+				(void) fsync_fname(basedir, true, progname);
+		}
+		else
+		{
+			(void) fsync_pgdata(basedir, progname);
+		}
 	}
 
 	if (verbose)
@@ -2001,6 +2007,7 @@ main(int argc, char **argv)
 		{"compress", required_argument, NULL, 'Z'},
 		{"label", required_argument, NULL, 'l'},
 		{"noclean", no_argument, NULL, 'n'},
+		{"nosync", no_argument, NULL, 'N'},
 		{"dbname", required_argument, NULL, 'd'},
 		{"host", required_argument, NULL, 'h'},
 		{"port", required_argument, NULL, 'p'},
@@ -2037,7 +2044,7 @@ main(int argc, char **argv)
 
 	atexit(cleanup_directories_atexit);
 
-	while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP",
+	while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nNzZ:d:c:h:p:U:s:S:wWvP",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
@@ -2115,6 +2122,9 @@ main(int argc, char **argv)
 			case 'n':
 				noclean = true;
 				break;
+			case 'N':
+				do_sync = false;
+				break;
 			case 'z':
 #ifdef HAVE_LIBZ
 				compresslevel = Z_DEFAULT_COMPRESSION;

src/bin/pg_basebackup/pg_receivexlog.c

@@ -336,6 +336,7 @@ StreamLog(void)
 	stream.stream_stop = stop_streaming;
 	stream.standby_message_timeout = standby_message_timeout;
 	stream.synchronous = synchronous;
+	stream.do_sync = true;
 	stream.mark_done = false;
 	stream.basedir = basedir;
 	stream.partial_suffix = ".partial";

src/bin/pg_basebackup/receivelog.c

@@ -41,8 +41,8 @@ static PGresult *HandleCopyStream(PGconn *conn, StreamCtl *stream,
 				 XLogRecPtr *stoppos);
 static int	CopyStreamPoll(PGconn *conn, long timeout_ms);
 static int	CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
-static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
-					XLogRecPtr blockpos, int64 *last_status);
+static bool ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf,
+					int len, XLogRecPtr blockpos, int64 *last_status);
 static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
 				   XLogRecPtr *blockpos);
 static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf,
@@ -56,7 +56,7 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
 						 uint32 *timeline);
 
 static bool
-mark_file_as_archived(const char *basedir, const char *fname)
+mark_file_as_archived(const char *basedir, const char *fname, bool do_sync)
 {
 	int			fd;
 	static char tmppath[MAXPGPATH];
@@ -74,10 +74,10 @@ mark_file_as_archived(const char *basedir, const char *fname)
 
 	close(fd);
 
-	if (fsync_fname(tmppath, false, progname) != 0)
+	if (do_sync && fsync_fname(tmppath, false, progname) != 0)
 		return false;
 
-	if (fsync_parent_path(tmppath, progname) != 0)
+	if (do_sync && fsync_parent_path(tmppath, progname) != 0)
 		return false;
 
 	return true;
@@ -134,9 +134,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
 		 * fsync, in case of a previous crash between padding and fsyncing the
 		 * file.
 		 */
-		if (fsync_fname(fn, false, progname) != 0)
+		if (stream->do_sync && fsync_fname(fn, false, progname) != 0)
 			return false;
-		if (fsync_parent_path(fn, progname) != 0)
+		if (stream->do_sync && fsync_parent_path(fn, progname) != 0)
 			return false;
 
 		return true;
@@ -173,9 +173,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
 	 * using synchronous mode, where the file is modified and fsynced
 	 * in-place, without a directory fsync.
 	 */
-	if (fsync_fname(fn, false, progname) != 0)
+	if (stream->do_sync && fsync_fname(fn, false, progname) != 0)
 		return false;
-	if (fsync_parent_path(fn, progname) != 0)
+	if (stream->do_sync && fsync_parent_path(fn, progname) != 0)
 		return false;
 
 	if (lseek(f, SEEK_SET, 0) != 0)
@@ -212,7 +212,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
 		return false;
 	}
 
-	if (fsync(walfile) != 0)
+	if (stream->do_sync && fsync(walfile) != 0)
 	{
 		fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
 				progname, current_walfile_name, strerror(errno));
@@ -258,7 +258,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
 	if (currpos == XLOG_SEG_SIZE && stream->mark_done)
 	{
 		/* writes error message if failed */
-		if (!mark_file_as_archived(stream->basedir, current_walfile_name))
+		if (!mark_file_as_archived(stream->basedir, current_walfile_name,
+								   stream->do_sync))
 			return false;
 	}
 
@@ -378,7 +379,8 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content)
 	if (stream->mark_done)
 	{
 		/* writes error message if failed */
-		if (!mark_file_as_archived(stream->basedir, histfname))
+		if (!mark_file_as_archived(stream->basedir, histfname,
+								   stream->do_sync))
 			return false;
 	}
 
@@ -836,7 +838,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream,
 		 */
 		if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1)
 		{
-			if (fsync(walfile) != 0)
+			if (stream->do_sync && fsync(walfile) != 0)
 			{
 				fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
 						progname, current_walfile_name, strerror(errno));
@@ -890,7 +892,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream,
 			/* Check the message type. */
 			if (copybuf[0] == 'k')
 			{
-				if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos,
+				if (!ProcessKeepaliveMsg(conn, stream, copybuf, r, blockpos,
 										 &last_status))
 					goto error;
 			}
@@ -1043,7 +1045,7 @@ CopyStreamReceive(PGconn *conn, long timeout, char **buffer)
  * Process the keepalive message.
  */
 static bool
-ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
+ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
 					XLogRecPtr blockpos, int64 *last_status)
 {
 	int			pos;
@@ -1079,7 +1081,7 @@ ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
 			 * data has been successfully replicated or not, at the normal
 			 * shutdown of the server.
 			 */
-			if (fsync(walfile) != 0)
+			if (stream->do_sync && fsync(walfile) != 0)
 			{
 				fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
 						progname, current_walfile_name, strerror(errno));

src/bin/pg_basebackup/receivelog.h

@@ -34,8 +34,10 @@ typedef struct StreamCtl
 								 * timeline */
 	int			standby_message_timeout;		/* Send status messages this
 												 * often */
-	bool		synchronous;	/* Flush data on write */
+	bool		synchronous;	/* Flush immediately WAL data on write */
 	bool		mark_done;		/* Mark segment as done in generated archive */
+	bool		do_sync;		/* Flush to disk to ensure consistent state
+								 * of data */
 
 	stream_stop_callback stream_stop;	/* Stop streaming when returns true */