Add progress reporting to pg_checksums

This adds a new option to pg_checksums called -P/--progress, showing
every second some information about the computation state of an
operation for --check and --enable (--disable only updates the control
file and is quick).  This requires a pre-scan of the data folder so as
the total size of checksummable items can be calculated, and then it
gets compared to the amount processed.

Similarly to what is done for pg_rewind and pg_basebackup, the
information printed in the progress report consists of the current
amount of data computed and the total amount of data to compute.  This
could be extended later on.

Author: Michael Banck, Bernd Helmle
Reviewed-by: Fabien Coelho, Michael Paquier
Discussion: https://postgr.es/m/1535719851.1286.17.camel@credativ.de

Author: michaelpq

doc/src/sgml/ref/pg_checksums.sgml

@@ -135,6 +135,17 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-P</option></term>
+      <term><option>--progress</option></term>
+      <listitem>
+       <para>
+        Enable progress reporting. Turning this on will deliver a progress
+        report while checking or enabling checksums.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
        <term><option>-V</option></term>
        <term><option>--version</option></term>

src/bin/pg_checksums/pg_checksums.c

@@ -15,6 +15,7 @@
 #include "postgres_fe.h"
 
 #include <dirent.h>
+#include <time.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
@@ -38,6 +39,7 @@ static ControlFileData *ControlFile;
 static char *only_relfilenode = NULL;
 static bool do_sync = true;
 static bool verbose = false;
+static bool showprogress = false;
 
 typedef enum
 {
@@ -60,6 +62,13 @@ static PgChecksumMode mode = PG_MODE_CHECK;
 
 static const char *progname;
 
+/*
+ * Progress status information.
+ */
+int64		total_size = 0;
+int64		current_size = 0;
+static pg_time_t last_progress_report = 0;
+
 static void
 usage(void)
 {
@@ -72,6 +81,7 @@ usage(void)
 	printf(_("  -d, --disable          disable data checksums\n"));
 	printf(_("  -e, --enable           enable data checksums\n"));
 	printf(_("  -N, --no-sync          do not wait for changes to be written safely to disk\n"));
+	printf(_("  -P, --progress         show progress information\n"));
 	printf(_("  -v, --verbose          output verbose messages\n"));
 	printf(_("  -r RELFILENODE         check only relation with specified relfilenode\n"));
 	printf(_("  -V, --version          output version information, then exit\n"));
@@ -98,6 +108,52 @@ static const char *const skip[] = {
 	NULL,
 };
 
+/*
+ * Report current progress status.  Parts borrowed from
+ * src/bin/pg_basebackup.c.
+ */
+static void
+progress_report(bool force)
+{
+	int			percent;
+	char		total_size_str[32];
+	char		current_size_str[32];
+	pg_time_t	now;
+
+	Assert(showprogress);
+
+	now = time(NULL);
+	if (now == last_progress_report && !force)
+		return;					/* Max once per second */
+
+	/* Save current time */
+	last_progress_report = now;
+
+	/* Adjust total size if current_size is larger */
+	if (current_size > total_size)
+		total_size = current_size;
+
+	/* Calculate current percentage of size done */
+	percent = total_size ? (int) ((current_size) * 100 / total_size) : 0;
+
+	snprintf(total_size_str, sizeof(total_size_str), INT64_FORMAT,
+			 total_size / (1024 * 1024));
+	snprintf(current_size_str, sizeof(current_size_str), INT64_FORMAT,
+			 current_size / (1024 * 1024));
+
+	/*
+	 * Separate step to keep platform-dependent format code out of
+	 * translatable strings.  And we only test for INT64_FORMAT availability
+	 * in snprintf, not fprintf.
+	 */
+	fprintf(stderr, "%*s/%s MB (%d%%) computed",
+			(int) strlen(current_size_str), current_size_str, total_size_str,
+			percent);
+
+	/* Stay on the same line if reporting to a terminal */
+	fprintf(stderr, isatty(fileno(stderr)) ? "\r" : "\n");
+}
+
 static bool
 skipfile(const char *fn)
 {
@@ -153,6 +209,7 @@ scan_file(const char *fn, BlockNumber segmentno)
 			continue;
 
 		csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE);
+		current_size += r;
 		if (mode == PG_MODE_CHECK)
 		{
 			if (csum != header->pd_checksum)
@@ -183,6 +240,9 @@ scan_file(const char *fn, BlockNumber segmentno)
 				exit(1);
 			}
 		}
+
+		if (showprogress)
+			progress_report(false);
 	}
 
 	if (verbose)
@@ -196,9 +256,17 @@ scan_file(const char *fn, BlockNumber segmentno)
 	close(f);
 }
 
-static void
-scan_directory(const char *basedir, const char *subdir)
+/*
+ * Scan the given directory for items which can be checksummed and
+ * operate on each one of them.  If "sizeonly" is true, the size of
+ * all the items which have checksums is computed and returned back
+ * to the caller without operating on the files.  This is used to compile
+ * the total size of the data directory for progress reports.
+ */
+static int64
+scan_directory(const char *basedir, const char *subdir, bool sizeonly)
 {
+	int64		dirsize = 0;
 	char		path[MAXPGPATH];
 	DIR		   *dir;
 	struct dirent *de;
@@ -275,16 +343,24 @@ scan_directory(const char *basedir, const char *subdir)
 				/* Relfilenode not to be included */
 				continue;
 
-			scan_file(fn, segmentno);
+			dirsize += st.st_size;
+
+			/*
+			 * No need to work on the file when calculating only the size of
+			 * the items in the data folder.
+			 */
+			if (!sizeonly)
+				scan_file(fn, segmentno);
 		}
 #ifndef WIN32
 		else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
 #else
 		else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
 #endif
-			scan_directory(path, de->d_name);
+			dirsize += scan_directory(path, de->d_name, sizeonly);
 	}
 	closedir(dir);
+	return dirsize;
 }
 
 int
@@ -296,6 +372,7 @@ main(int argc, char *argv[])
 		{"disable", no_argument, NULL, 'd'},
 		{"enable", no_argument, NULL, 'e'},
 		{"no-sync", no_argument, NULL, 'N'},
+		{"progress", no_argument, NULL, 'P'},
 		{"verbose", no_argument, NULL, 'v'},
 		{NULL, 0, NULL, 0}
 	};
@@ -323,7 +400,7 @@ main(int argc, char *argv[])
 		}
 	}
 
-	while ((c = getopt_long(argc, argv, "cD:deNr:v", long_options, &option_index)) != -1)
+	while ((c = getopt_long(argc, argv, "cD:deNPr:v", long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
@@ -353,6 +430,9 @@ main(int argc, char *argv[])
 				}
 				only_relfilenode = pstrdup(optarg);
 				break;
+			case 'P':
+				showprogress = true;
+				break;
 			default:
 				fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
 				exit(1);
@@ -447,9 +527,27 @@ main(int argc, char *argv[])
 	/* Operate on all files if checking or enabling checksums */
 	if (mode == PG_MODE_CHECK || mode == PG_MODE_ENABLE)
 	{
-		scan_directory(DataDir, "global");
-		scan_directory(DataDir, "base");
-		scan_directory(DataDir, "pg_tblspc");
+		/*
+		 * If progress status information is requested, we need to scan the
+		 * directory tree twice: once to know how much total data needs to be
+		 * processed and once to do the real work.
+		 */
+		if (showprogress)
+		{
+			total_size = scan_directory(DataDir, "global", true);
+			total_size += scan_directory(DataDir, "base", true);
+			total_size += scan_directory(DataDir, "pg_tblspc", true);
+		}
+
+		(void) scan_directory(DataDir, "global", false);
+		(void) scan_directory(DataDir, "base", false);
+		(void) scan_directory(DataDir, "pg_tblspc", false);
+
+		if (showprogress)
+		{
+			progress_report(true);
+			fprintf(stderr, "\n");	/* Need to move to next line */
+		}
 
 		printf(_("Checksum operation completed\n"));
 		printf(_("Files scanned:  %s\n"), psprintf(INT64_FORMAT, files));