pg_basebackup: Add support for relocating tablespaces

Tablespaces can be relocated in plain backup mode by specifying one or
more -T olddir=newdir options.

Author: Steeve Lennmark <steevel@handeldsbanken.se>
Reviewed-by: Peter Eisentraut <peter_e@gmx.net>

Author: petere

doc/src/sgml/ref/pg_basebackup.sgml

@@ -202,6 +202,33 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-T <replaceable class="parameter">olddir</replaceable>=<replaceable class="parameter">newdir</replaceable></option></term>
+      <term><option>--tablespace-mapping=<replaceable class="parameter">olddir</replaceable>=<replaceable class="parameter">newdir</replaceable></option></term>
+      <listitem>
+       <para>
+        Relocate the tablespace in directory <replaceable>olddir</replaceable>
+        to <replaceable>newdir</replaceable> during the backup.  To be
+        effective, <replaceable>olddir</replaceable> must exactly match the
+        path specification of the tablespace as it is currently defined.  (But
+        it is not an error if there is no tablespace
+        in <replaceable>olddir</replaceable> contained in the backup.)
+        Both <replaceable>olddir</replaceable>
+        and <replaceable>newdir</replaceable> must be absolute paths.  If a
+        path happens to contain a <literal>=</literal> sign, escape it with a
+        backslash.  This option can be specified multiple times for multiple
+        tablespaces.  See examples below.
+       </para>
+
+       <para>
+        If a tablespace is relocated in this way, the symbolic links inside
+        the main data directory are updated to point to the new location.  So
+        the new data directory is ready to be used for a new server instance
+        with all tablespaces in the updated locations.
+        </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>--xlogdir=<replaceable class="parameter">xlogdir</replaceable></option></term>
       <listitem>
@@ -528,9 +555,13 @@ PostgreSQL documentation
   </para>
 
   <para>
-   The way <productname>PostgreSQL</productname> manages tablespaces, the path
-   for all additional tablespaces must be identical whenever a backup is
-   restored. The main data directory, however, is relocatable to any location.
+   Tablespaces will in plain format by default be backed up to the same path
+   they have on the server, unless the
+   option <replaceable>--tablespace-mapping</replaceable> is used.  Without
+   this option, running a plain format base backup on the same host as the
+   server will not work if tablespaces are in use, because the backup would
+   have to be written to the same directory locations as the original
+   tablespaces.
   </para>
 
   <para>
@@ -570,6 +601,15 @@ PostgreSQL documentation
    (This command will fail if there are multiple tablespaces in the
    database.)
   </para>
+
+  <para>
+   To create a backup of a local database where the tablespace in
+   <filename>/opt/ts</filename> is relocated
+   to <filename>./backup/ts</filename>:
+<screen>
+<prompt>$</prompt> <userinput>pg_basebackup -D backup/data -T /opt/ts=$(pwd)/backup/ts</userinput>
+</screen>
+  </para>
  </refsect1>
 
  <refsect1>

src/bin/pg_basebackup/pg_basebackup.c

@@ -35,8 +35,24 @@
 #include "streamutil.h"
 
 
+#define atooid(x)  ((Oid) strtoul((x), NULL, 10))
+
+typedef struct TablespaceListCell
+{
+	struct TablespaceListCell *next;
+	char old_dir[MAXPGPATH];
+	char new_dir[MAXPGPATH];
+} TablespaceListCell;
+
+typedef struct TablespaceList
+{
+	TablespaceListCell *head;
+	TablespaceListCell *tail;
+} TablespaceList;
+
 /* Global options */
 static char *basedir = NULL;
+static TablespaceList tablespace_dirs = {NULL, NULL};
 static char *xlog_dir = "";
 static char	format = 'p';		/* p(lain)/t(ar) */
 static char *label = "pg_basebackup base backup";
@@ -90,6 +106,10 @@ static void BaseBackup(void);
 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
 					 bool segment_finished);
 
+static const char *get_tablespace_mapping(const char *dir);
+static void update_tablespace_symlink(Oid oid, const char *old_dir);
+static void tablespace_list_append(const char *arg);
+
 
 static void disconnect_and_exit(int code)
 {
@@ -110,6 +130,77 @@ static void disconnect_and_exit(int code)
 }
 
 
+/*
+ * Split argument into old_dir and new_dir and append to tablespace mapping
+ * list.
+ */
+static void
+tablespace_list_append(const char *arg)
+{
+	TablespaceListCell *cell = (TablespaceListCell *) pg_malloc0(sizeof(TablespaceListCell));
+	char		*dst;
+	char		*dst_ptr;
+	const char	*arg_ptr;
+
+	dst_ptr = dst = cell->old_dir;
+	for (arg_ptr = arg; *arg_ptr; arg_ptr++)
+	{
+		if (dst_ptr - dst >= MAXPGPATH)
+		{
+			fprintf(stderr, _("%s: directory name too long\n"),	progname);
+			exit(1);
+		}
+
+		if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
+			;  /* skip backslash escaping = */
+		else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
+		{
+			if (*cell->new_dir)
+			{
+				fprintf(stderr, _("%s: multiple \"=\" signs in tablespace mapping\n"), progname);
+				exit(1);
+			}
+			else
+				dst = dst_ptr = cell->new_dir;
+		}
+		else
+			*dst_ptr++ = *arg_ptr;
+	}
+
+	if (!*cell->old_dir || !*cell->new_dir)
+	{
+		fprintf(stderr,
+				_("%s: invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"\n"),
+				progname, arg);
+		exit(1);
+	}
+
+	/* This check isn't absolutely necessary.  But all tablespaces are created
+	 * with absolute directories, so specifying a non-absolute path here would
+	 * just never match, possibly confusing users.  It's also good to be
+	 * consistent with the new_dir check. */
+	if (!is_absolute_path(cell->old_dir))
+	{
+		fprintf(stderr, _("%s: old directory not absolute in tablespace mapping: %s\n"),
+				progname, cell->old_dir);
+		exit(1);
+	}
+
+	if (!is_absolute_path(cell->new_dir))
+	{
+		fprintf(stderr, _("%s: new directory not absolute in tablespace mapping: %s\n"),
+				progname, cell->new_dir);
+		exit(1);
+	}
+
+	if (tablespace_dirs.tail)
+		tablespace_dirs.tail->next = cell;
+	else
+		tablespace_dirs.head = cell;
+	tablespace_dirs.tail = cell;
+}
+
+
 #ifdef HAVE_LIBZ
 static const char *
 get_gz_error(gzFile gzf)
@@ -137,6 +228,8 @@ usage(void)
 	printf(_("  -F, --format=p|t       output format (plain (default), tar)\n"));
 	printf(_("  -R, --write-recovery-conf\n"
 			 "                         write recovery.conf after backup\n"));
+	printf(_("  -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
+			 "                         relocate tablespace in OLDDIR to NEWDIR\n"));
 	printf(_("  -x, --xlog             include required WAL files in backup (fetch mode)\n"));
 	printf(_("  -X, --xlog-method=fetch|stream\n"
 			 "                         include required WAL files with specified method\n"));
@@ -899,15 +992,60 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
 		PQfreemem(copybuf);
 }
 
+
+/*
+ * Retrieve tablespace path, either relocated or original depending on whether
+ * -T was passed or not.
+ */
+static const char *
+get_tablespace_mapping(const char *dir)
+{
+	TablespaceListCell *cell;
+
+	for (cell = tablespace_dirs.head; cell; cell = cell->next)
+		if (strcmp(dir, cell->old_dir) == 0)
+			return cell->new_dir;
+
+	return dir;
+}
+
+
+/*
+ * Update symlinks to reflect relocated tablespace.
+ */
+static void
+update_tablespace_symlink(Oid oid, const char *old_dir)
+{
+	const char *new_dir = get_tablespace_mapping(old_dir);
+
+	if (strcmp(old_dir, new_dir) != 0)
+	{
+		char *linkloc = psprintf("%s/pg_tblspc/%d", basedir, oid);
+
+		if (unlink(linkloc) != 0 && errno != ENOENT)
+		{
+			fprintf(stderr, _("%s: could not remove symbolic link \"%s\": %s"),
+					progname, linkloc, strerror(errno));
+			disconnect_and_exit(1);
+		}
+		if (symlink(new_dir, linkloc) != 0)
+		{
+			fprintf(stderr, _("%s: could not create symbolic link \"%s\": %s"),
+					progname, linkloc, strerror(errno));
+			disconnect_and_exit(1);
+		}
+	}
+}
+
+
 /*
  * Receive a tar format stream from the connection to the server, and unpack
  * the contents of it into a directory. Only files, directories and
  * symlinks are supported, no other kinds of special files.
  *
  * If the data is for the main data directory, it will be restored in the
  * specified directory. If it's for another tablespace, it will be restored
- * in the original directory, since relocation of tablespaces is not
- * supported.
+ * in the original or mapped directory.
  */
 static void
 ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
@@ -923,7 +1061,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
 	if (basetablespace)
 		strlcpy(current_path, basedir, sizeof(current_path));
 	else
-		strlcpy(current_path, PQgetvalue(res, rownum, 1), sizeof(current_path));
+		strlcpy(current_path, get_tablespace_mapping(PQgetvalue(res, rownum, 1)), sizeof(current_path));
 
 	/*
 	 * Get the COPY data
@@ -1503,7 +1641,10 @@ BaseBackup(void)
 		 * we do anything anyway.
 		 */
 		if (format == 'p' && !PQgetisnull(res, i, 1))
-			verify_dir_is_empty_or_create(PQgetvalue(res, i, 1));
+		{
+			char *path = (char *) get_tablespace_mapping(PQgetvalue(res, i, 1));
+			verify_dir_is_empty_or_create(path);
+		}
 	}
 
 	/*
@@ -1545,6 +1686,17 @@ BaseBackup(void)
 		progress_report(PQntuples(res), NULL, true);
 		fprintf(stderr, "\n");	/* Need to move to next line */
 	}
+
+	if (format == 'p' && tablespace_dirs.head != NULL)
+	{
+		for (i = 0; i < PQntuples(res); i++)
+		{
+			Oid tblspc_oid = atooid(PQgetvalue(res, i, 0));
+			if (tblspc_oid)
+				update_tablespace_symlink(tblspc_oid, PQgetvalue(res, i, 1));
+		}
+	}
+
 	PQclear(res);
 
 	/*
@@ -1696,6 +1848,7 @@ main(int argc, char **argv)
 		{"format", required_argument, NULL, 'F'},
 		{"checkpoint", required_argument, NULL, 'c'},
 		{"write-recovery-conf", no_argument, NULL, 'R'},
+		{"tablespace-mapping", required_argument, NULL, 'T'},
 		{"xlog", no_argument, NULL, 'x'},
 		{"xlog-method", required_argument, NULL, 'X'},
 		{"gzip", no_argument, NULL, 'z'},
@@ -1735,7 +1888,7 @@ main(int argc, char **argv)
 		}
 	}
 
-	while ((c = getopt_long(argc, argv, "D:F:RxX:l:zZ:d:c:h:p:U:s:wWvP",
+	while ((c = getopt_long(argc, argv, "D:F:RT:xX:l:zZ:d:c:h:p:U:s:wWvP",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
@@ -1759,6 +1912,9 @@ main(int argc, char **argv)
 			case 'R':
 				writerecoveryconf = true;
 				break;
+			case 'T':
+				tablespace_list_append(optarg);
+				break;
 			case 'x':
 				if (includewal)
 				{