Skip to content

Commit e65497d

Browse files
MasaoFujiiMasaoFujii
committed
Report progress of streaming base backup.
This commit adds pg_stat_progress_basebackup view that reports the progress while an application like pg_basebackup is taking a base backup. This uses the progress reporting infrastructure added by c16dc1a, adding support for streaming base backup. Bump catversion. Author: Fujii Masao Reviewed-by: Kyotaro Horiguchi, Amit Langote, Sergei Kornilov Discussion: https://postgr.es/m/9ed8b801-8215-1f3d-62d7-65bff53f6e94@oss.nttdata.com
1 parent d79fb88 commit e65497d

File tree

11 files changed

+339
-6
lines changed

11 files changed

+339
-6
lines changed

‎doc/src/sgml/monitoring.sgml‎

Lines changed: 162 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
376376
</entry>
377377
</row>
378378

379+
<row>
380+
<entry><structname>pg_stat_progress_basebackup</structname><indexterm><primary>pg_stat_progress_basebackup</primary></indexterm></entry>
381+
<entry>One row for each WAL sender process streaming a base backup,
382+
showing current progress.
383+
See <xref linkend='basebackup-progress-reporting'/>.
384+
</entry>
385+
</row>
386+
379387
</tbody>
380388
</tgroup>
381389
</table>
@@ -3535,7 +3543,10 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
35353543
certain commands during command execution. Currently, the only commands
35363544
which support progress reporting are <command>ANALYZE</command>,
35373545
<command>CLUSTER</command>,
3538-
<command>CREATE INDEX</command>, and <command>VACUUM</command>.
3546+
<command>CREATE INDEX</command>, <command>VACUUM</command>,
3547+
and <xref linkend="protocol-replication-base-backup"/> (i.e., replication
3548+
command that <xref linkend="app-pgbasebackup"/> issues to take
3549+
a base backup).
35393550
This may be expanded in the future.
35403551
</para>
35413552

@@ -4336,6 +4347,156 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
43364347
</tbody>
43374348
</tgroup>
43384349
</table>
4350+
</sect2>
4351+
4352+
<sect2 id="basebackup-progress-reporting">
4353+
<title>Base Backup Progress Reporting</title>
4354+
4355+
<para>
4356+
Whenever an application like <application>pg_basebackup</application>
4357+
is taking a base backup, the
4358+
<structname>pg_stat_progress_basebackup</structname>
4359+
view will contain a row for each WAL sender process that is currently
4360+
running <command>BASE_BACKUP</command> replication command
4361+
and streaming the backup. The tables below describe the information
4362+
that will be reported and provide information about how to interpret it.
4363+
</para>
4364+
4365+
<table id="pg-stat-progress-basebackup-view" xreflabel="pg_stat_progress_basebackup">
4366+
<title><structname>pg_stat_progress_basebackup</structname> View</title>
4367+
<tgroup cols="3">
4368+
<thead>
4369+
<row>
4370+
<entry>Column</entry>
4371+
<entry>Type</entry>
4372+
<entry>Description</entry>
4373+
</row>
4374+
</thead>
4375+
4376+
<tbody>
4377+
<row>
4378+
<entry><structfield>pid</structfield></entry>
4379+
<entry><type>integer</type></entry>
4380+
<entry>Process ID of a WAL sender process.</entry>
4381+
</row>
4382+
<row>
4383+
<entry><structfield>phase</structfield></entry>
4384+
<entry><type>text</type></entry>
4385+
<entry>Current processing phase. See <xref linkend="basebackup-phases" />.</entry>
4386+
</row>
4387+
<row>
4388+
<entry><structfield>backup_total</structfield></entry>
4389+
<entry><type>bigint</type></entry>
4390+
<entry>
4391+
Total amount of data that will be streamed. If progress reporting
4392+
is not enabled in <application>pg_basebackup</application>
4393+
(i.e., <literal>--progress</literal> option is not specified),
4394+
this is <literal>0</literal>. Otherwise, this is estimated and
4395+
reported as of the beginning of
4396+
<literal>streaming database files</literal> phase. Note that
4397+
this is only an approximation since the database
4398+
may change during <literal>streaming database files</literal> phase
4399+
and WAL log may be included in the backup later. This is always
4400+
the same value as <structfield>backup_streamed</structfield>
4401+
once the amount of data streamed exceeds the estimated
4402+
total size.
4403+
</entry>
4404+
</row>
4405+
<row>
4406+
<entry><structfield>backup_streamed</structfield></entry>
4407+
<entry><type>bigint</type></entry>
4408+
<entry>
4409+
Amount of data streamed. This counter only advances
4410+
when the phase is <literal>streaming database files</literal> or
4411+
<literal>transfering wal files</literal>.
4412+
</entry>
4413+
</row>
4414+
<row>
4415+
<entry><structfield>tablespaces_total</structfield></entry>
4416+
<entry><type>bigint</type></entry>
4417+
<entry>
4418+
Total number of tablespaces that will be streamed.
4419+
</entry>
4420+
</row>
4421+
<row>
4422+
<entry><structfield>tablespaces_streamed</structfield></entry>
4423+
<entry><type>bigint</type></entry>
4424+
<entry>
4425+
Number of tablespaces streamed. This counter only
4426+
advances when the phase is <literal>streaming database files</literal>.
4427+
</entry>
4428+
</row>
4429+
</tbody>
4430+
</tgroup>
4431+
</table>
4432+
4433+
<table id="basebackup-phases">
4434+
<title>Base backup phases</title>
4435+
<tgroup cols="2">
4436+
<thead>
4437+
<row>
4438+
<entry>Phase</entry>
4439+
<entry>Description</entry>
4440+
</row>
4441+
</thead>
4442+
<tbody>
4443+
<row>
4444+
<entry><literal>initializing</literal></entry>
4445+
<entry>
4446+
The WAL sender process is preparing to begin the backup.
4447+
This phase is expected to be very brief.
4448+
</entry>
4449+
</row>
4450+
<row>
4451+
<entry><literal>waiting for checkpoint to finish</literal></entry>
4452+
<entry>
4453+
The WAL sender process is currently performing
4454+
<function>pg_start_backup</function> to set up for
4455+
taking a base backup, and waiting for backup start
4456+
checkpoint to finish.
4457+
</entry>
4458+
</row>
4459+
<row>
4460+
<entry><literal>estimating backup size</literal></entry>
4461+
<entry>
4462+
The WAL sender process is currently estimating the total amount
4463+
of database files that will be streamed as a base backup.
4464+
</entry>
4465+
</row>
4466+
<row>
4467+
<entry><literal>streaming database files</literal></entry>
4468+
<entry>
4469+
The WAL sender process is currently streaming database files
4470+
as a base backup.
4471+
</entry>
4472+
</row>
4473+
<row>
4474+
<entry><literal>waiting for wal archiving to finish</literal></entry>
4475+
<entry>
4476+
The WAL sender process is currently performing
4477+
<function>pg_stop_backup</function> to finish the backup,
4478+
and waiting for all the WAL files required for the base backup
4479+
to be successfully archived.
4480+
If either <literal>--wal-method=none</literal> or
4481+
<literal>--wal-method=stream</literal> is specified in
4482+
<application>pg_basebackup</application>, the backup will end
4483+
when this phase is completed.
4484+
</entry>
4485+
</row>
4486+
<row>
4487+
<entry><literal>transferring wal files</literal></entry>
4488+
<entry>
4489+
The WAL sender process is currently transferring all WAL logs
4490+
generated during the backup. This phase occurs after
4491+
<literal>waiting for wal archiving to finish</literal> phase if
4492+
<literal>--wal-method=fetch</literal> is specified in
4493+
<application>pg_basebackup</application>. The backup will end
4494+
when this phase is completed.
4495+
</entry>
4496+
</row>
4497+
</tbody>
4498+
</tgroup>
4499+
</table>
43394500

43404501
</sect2>
43414502
</sect1>

‎doc/src/sgml/protocol.sgml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2465,7 +2465,7 @@ The commands accepted in replication mode are:
24652465
</listitem>
24662466
</varlistentry>
24672467

2468-
<varlistentry>
2468+
<varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
24692469
<term><literal>BASE_BACKUP</literal> [ <literal>LABEL</literal> <replaceable>'label'</replaceable> ] [ <literal>PROGRESS</literal> ] [ <literal>FAST</literal> ] [ <literal>WAL</literal> ] [ <literal>NOWAIT</literal> ] [ <literal>MAX_RATE</literal> <replaceable>rate</replaceable> ] [ <literal>TABLESPACE_MAP</literal> ] [ <literal>NOVERIFY_CHECKSUMS</literal> ]
24702470
<indexterm><primary>BASE_BACKUP</primary></indexterm>
24712471
</term>

‎doc/src/sgml/ref/pg_basebackup.sgml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,13 @@ PostgreSQL documentation
104104
</listitem>
105105
</itemizedlist>
106106
</para>
107+
108+
<para>
109+
Whenever <application>pg_basebackup</application> is taking a base
110+
backup, the <structname>pg_stat_progress_basebackup</structname>
111+
view will report the progress of the backup.
112+
See <xref linkend="basebackup-progress-reporting"/> for details.
113+
</para>
107114
</refsect1>
108115

109116
<refsect1>
@@ -459,6 +466,15 @@ PostgreSQL documentation
459466
This may make the backup take slightly longer, and in particular it
460467
will take longer before the first data is sent.
461468
</para>
469+
<para>
470+
Whether this is enabled or not, the
471+
<structname>pg_stat_progress_basebackup</structname> view
472+
report the progress of the backup in the server side. But note
473+
that the total amount of data that will be streamed is estimated
474+
and reported only when this option is enabled. In other words,
475+
<literal>backup_total</literal> column in the view always
476+
indicates <literal>0</literal> if this option is disabled.
477+
</para>
462478
</listitem>
463479
</varlistentry>
464480

‎src/backend/access/transam/xlog.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
#include "catalog/catversion.h"
4040
#include "catalog/pg_control.h"
4141
#include "catalog/pg_database.h"
42+
#include "commands/progress.h"
4243
#include "commands/tablespace.h"
4344
#include "common/controldata_utils.h"
4445
#include "miscadmin.h"
@@ -10228,6 +10229,10 @@ issue_xlog_fsync(int fd, XLogSegNo segno)
1022810229
* active at the same time, and they don't conflict with an exclusive backup
1022910230
* either.
1023010231
*
10232+
* tablespaces is required only when this function is called while
10233+
* the streaming base backup requested by pg_basebackup is running.
10234+
* NULL should be specified otherwise.
10235+
*
1023110236
* tblspcmapfile is required mainly for tar format in windows as native windows
1023210237
* utilities are not able to create symlinks while extracting files from tar.
1023310238
* However for consistency, the same is used for all platforms.
@@ -10470,6 +10475,14 @@ do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
1047010475

1047110476
datadirpathlen = strlen(DataDir);
1047210477

10478+
/*
10479+
* Report that we are now estimating the total backup size
10480+
* if we're streaming base backup as requested by pg_basebackup
10481+
*/
10482+
if (tablespaces)
10483+
pgstat_progress_update_param(PROGRESS_BASEBACKUP_PHASE,
10484+
PROGRESS_BASEBACKUP_PHASE_ESTIMATE_BACKUP_SIZE);
10485+
1047310486
/* Collect information about all tablespaces */
1047410487
tblspcdir = AllocateDir("pg_tblspc");
1047510488
while ((de = ReadDir(tblspcdir, "pg_tblspc")) != NULL)

‎src/backend/catalog/system_views.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1060,6 +1060,22 @@ CREATE VIEW pg_stat_progress_create_index AS
10601060
FROM pg_stat_get_progress_info('CREATE INDEX') AS S
10611061
LEFT JOIN pg_database D ON S.datid = D.oid;
10621062

1063+
CREATE VIEW pg_stat_progress_basebackup AS
1064+
SELECT
1065+
S.pid AS pid,
1066+
CASE S.param1 WHEN 0 THEN 'initializing'
1067+
WHEN 1 THEN 'waiting for checkpoint to finish'
1068+
WHEN 2 THEN 'estimating backup size'
1069+
WHEN 3 THEN 'streaming database files'
1070+
WHEN 4 THEN 'waiting for wal archiving to finish'
1071+
WHEN 5 THEN 'transferring wal files'
1072+
END AS phase,
1073+
S.param2 AS backup_total,
1074+
S.param3 AS backup_streamed,
1075+
S.param4 AS tablespaces_total,
1076+
S.param5 AS tablespaces_streamed
1077+
FROM pg_stat_get_progress_info('BASEBACKUP') AS S;
1078+
10631079
CREATE VIEW pg_user_mappings AS
10641080
SELECT
10651081
U.oid AS umid,

0 commit comments

Comments
 (0)