diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f99d0f27..c3ad89568 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,8 +72,7 @@ jobs: - name: Install Testgres run: | git clone -b no-port-for --single-branch --depth 1 https://github.com/postgrespro/testgres.git - cd testgres - python setup.py install + pip3 install psycopg2 ./testgres # Grant the Github runner user full control of the workspace for initdb to successfully process the data folder - name: Test Probackup diff --git a/LICENSE b/LICENSE index 66476e8a9..4fed760f8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2023, Postgres Professional +Copyright (c) 2015-2025, Postgres Professional Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group diff --git a/README.md b/README.md index 2279b97a4..16aec1dcc 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 11, 12, 13, 14, 15, 16 +* PostgreSQL 13, 14, 15, 16, 17 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -41,9 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL 11, 12, 13, 14, 15, 16 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13, 14, 15, 16 -* Postgres Pro Enterprise 11, 12, 13, 14, 15, 16 +* vanilla PostgreSQL 13, 14, 15, 16, 17 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 13, 14, 15, 16, 17 +* Postgres Pro Enterprise 13, 14, 15, 16, 17 ## Limitations diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 1491059c5..4f05da26a 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -529,14 +529,14 @@ doc/src/sgml/pgprobackup.sgml Initialize the backup catalog: -backup_user@backup_host:~$ pg_probackup init -B /mnt/backups +backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully initialized Add a backup instance called mydb to the backup catalog: -backup_user@backup_host:~$ pg_probackup add-instance \ +backup_user@backup_host:~$ pg_probackup-16 add-instance \ -B /mnt/backups \ -D /var/lib/pgpro/std-16/data \ --instance=node \ @@ -548,7 +548,7 @@ INFO: Instance 'node' successfully initialized Make a FULL backup: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b FULL \ --instance=node \ @@ -582,7 +582,7 @@ INFO: Backup SCUN1Q completed List the backups of the instance: -backup_user@backup_host:~$ pg_probackup show \ +backup_user@backup_host:~$ pg_probackup-16 show \ -B /mnt/backups \ --instance=node ================================================================================================================================ @@ -594,7 +594,7 @@ backup_user@backup_host:~$ pg_probackup show \ Make an incremental backup in the DELTA mode: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b DELTA \ --instance=node \ @@ -631,7 +631,7 @@ INFO: Backup SCUN22 completed Add or modify some parameters in the pg_probackup configuration file, so that you do not have to specify them each time on the command line: -backup_user@backup_host:~$ pg_probackup set-config \ +backup_user@backup_host:~$ pg_probackup-16 set-config \ -B /mnt/backups \ --instance=node \ --remote-host=postgres_host \ @@ -643,7 +643,7 @@ backup_user@backup_host:~$ pg_probackup set-config \ Check the configuration of the instance: -backup_user@backup_host:~$ pg_probackup show-config \ +backup_user@backup_host:~$ pg_probackup-16 show-config \ -B /mnt/backups \ --instance=node # Backup instance information @@ -686,7 +686,7 @@ remote-user = postgres Make another incremental backup in the DELTA mode, omitting the parameters stored in the configuration file earlier: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b DELTA \ --instance=node \ @@ -718,7 +718,7 @@ INFO: Backup SCUN2C completed List the backups of the instance again: -backup_user@backup_host:~$ pg_probackup show \ +backup_user@backup_host:~$ pg_probackup-16 show \ -B /mnt/backups \ --instance=node =================================================================================================================================== @@ -732,7 +732,7 @@ backup_user@backup_host:~$ pg_probackup show \ Restore the data from the latest available backup to an arbitrary location: -backup_user@backup_host:~$ pg_probackup restore \ +backup_user@backup_host:~$ pg_probackup-16 restore \ -B /mnt/backups \ -D /var/lib/pgpro/std-16/staging-data \ --instance=node @@ -924,7 +924,7 @@ yumdownloader --source pg_probackup-16 . /etc/os-release -echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p$VERSION_ID x86_64 vanilla" | \ +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p${VERSION_ID%%.*} x86_64 vanilla" | \ sudo tee /etc/apt/sources.list.d/pg_probackup.list @@ -934,7 +934,7 @@ sudo tee /etc/apt/sources.list.d/pg_probackup.list . /etc/os-release -echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-$VERSION_ID x86_64 vanilla" | \ +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-${VERSION_ID%%.*} x86_64 vanilla" | \ sudo tee /etc/apt/sources.list.d/pg_probackup.list @@ -2332,7 +2332,7 @@ pg_probackup restore -B backup_dir --instance=rel_path); - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; join_path_components(from_fullpath, instance_config.pgdata, src_pg_control_file->rel_path); join_path_components(to_fullpath, current.database_dir, src_pg_control_file->rel_path); @@ -947,7 +946,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, /* * Confirm that this server version is supported */ -static void +void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) { PGresult *res = NULL; diff --git a/src/catalog.c b/src/catalog.c index b29090789..409d9141f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -3,7 +3,7 @@ * catalog.c: backup catalog operation * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ @@ -1755,16 +1755,16 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) for (i = 0; i < parray_num(timelineinfos); i++) { - timelineInfo *tlinfo = parray_get(timelineinfos, i); + timelineInfo *tlInfo = parray_get(timelineinfos, i); for (j = 0; j < parray_num(backups); j++) { pgBackup *backup = parray_get(backups, j); - if (tlinfo->tli == backup->tli) + if (tlInfo->tli == backup->tli) { - if (tlinfo->backups == NULL) - tlinfo->backups = parray_new(); + if (tlInfo->backups == NULL) + tlInfo->backups = parray_new(); - parray_append(tlinfo->backups, backup); + parray_append(tlInfo->backups, backup); } } } @@ -1772,10 +1772,10 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) /* determine oldest backup and closest backup for every timeline */ for (i = 0; i < parray_num(timelineinfos); i++) { - timelineInfo *tlinfo = parray_get(timelineinfos, i); + timelineInfo *tlInfo = parray_get(timelineinfos, i); - tlinfo->oldest_backup = get_oldest_backup(tlinfo); - tlinfo->closest_backup = get_closest_backup(tlinfo); + tlInfo->oldest_backup = get_oldest_backup(tlInfo); + tlInfo->closest_backup = get_closest_backup(tlInfo); } /* determine which WAL segments must be kept because of wal retention */ @@ -1845,18 +1845,18 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) for (i = 0; i < parray_num(timelineinfos); i++) { int count = 0; - timelineInfo *tlinfo = parray_get(timelineinfos, i); + timelineInfo *tlInfo = parray_get(timelineinfos, i); /* * Iterate backward on backups belonging to this timeline to find * anchor_backup. NOTE Here we rely on the fact that backups list * is ordered by start_lsn DESC. */ - if (tlinfo->backups) + if (tlInfo->backups) { - for (j = 0; j < parray_num(tlinfo->backups); j++) + for (j = 0; j < parray_num(tlInfo->backups); j++) { - pgBackup *backup = parray_get(tlinfo->backups, j); + pgBackup *backup = parray_get(tlInfo->backups, j); /* sanity */ if (XLogRecPtrIsInvalid(backup->start_lsn) || @@ -1886,12 +1886,12 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) if (count == instance->wal_depth) { elog(LOG, "On timeline %i WAL is protected from purge at %X/%X", - tlinfo->tli, + tlInfo->tli, (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn)); - tlinfo->anchor_lsn = backup->start_lsn; - tlinfo->anchor_tli = backup->tli; + tlInfo->anchor_lsn = backup->start_lsn; + tlInfo->anchor_tli = backup->tli; break; } } @@ -1916,7 +1916,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) * If closest_backup is not available, then general WAL purge rules * are applied. */ - if (XLogRecPtrIsInvalid(tlinfo->anchor_lsn)) + if (XLogRecPtrIsInvalid(tlInfo->anchor_lsn)) { /* * Failed to find anchor_lsn in our own timeline. @@ -1942,7 +1942,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) xlogInterval *interval = NULL; TimeLineID tli = 0; /* check if tli has closest_backup */ - if (!tlinfo->closest_backup) + if (!tlInfo->closest_backup) /* timeline has no closest_backup, wal retention cannot be * applied to this timeline. * Timeline will be purged up to oldest_backup if any or @@ -1952,47 +1952,47 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) continue; /* sanity for closest_backup */ - if (XLogRecPtrIsInvalid(tlinfo->closest_backup->start_lsn) || - tlinfo->closest_backup->tli <= 0) + if (XLogRecPtrIsInvalid(tlInfo->closest_backup->start_lsn) || + tlInfo->closest_backup->tli <= 0) continue; /* * Set anchor_lsn and anchor_tli to protect whole timeline from purge * In the example above: tli3. */ - tlinfo->anchor_lsn = tlinfo->closest_backup->start_lsn; - tlinfo->anchor_tli = tlinfo->closest_backup->tli; + tlInfo->anchor_lsn = tlInfo->closest_backup->start_lsn; + tlInfo->anchor_tli = tlInfo->closest_backup->tli; /* closest backup may be located not in parent timeline */ - closest_backup = tlinfo->closest_backup; + closest_backup = tlInfo->closest_backup; - tli = tlinfo->tli; + tli = tlInfo->tli; /* * Iterate over parent timeline chain and * look for timeline where closest_backup belong */ - while (tlinfo->parent_link) + while (tlInfo->parent_link) { /* In case of intermediate timeline save to keep_segments * begin_segno and switchpoint segment. * In case of final timelines save to keep_segments * closest_backup start_lsn segment and switchpoint segment. */ - XLogRecPtr switchpoint = tlinfo->switchpoint; + XLogRecPtr switchpoint = tlInfo->switchpoint; - tlinfo = tlinfo->parent_link; + tlInfo = tlInfo->parent_link; - if (tlinfo->keep_segments == NULL) - tlinfo->keep_segments = parray_new(); + if (tlInfo->keep_segments == NULL) + tlInfo->keep_segments = parray_new(); /* in any case, switchpoint segment must be added to interval */ interval = palloc(sizeof(xlogInterval)); GetXLogSegNo(switchpoint, interval->end_segno, instance->xlog_seg_size); /* Save [S1`, S2] to keep_segments */ - if (tlinfo->tli != closest_backup->tli) - interval->begin_segno = tlinfo->begin_segno; + if (tlInfo->tli != closest_backup->tli) + interval->begin_segno = tlInfo->begin_segno; /* Save [B1, S1] to keep_segments */ else GetXLogSegNo(closest_backup->start_lsn, interval->begin_segno, instance->xlog_seg_size); @@ -2002,27 +2002,27 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) * covered by other larger interval. */ - GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size); - GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size); + GetXLogFileName(begin_segno_str, tlInfo->tli, interval->begin_segno, instance->xlog_seg_size); + GetXLogFileName(end_segno_str, tlInfo->tli, interval->end_segno, instance->xlog_seg_size); elog(LOG, "Timeline %i to stay reachable from timeline %i " "protect from purge WAL interval between " "%s and %s on timeline %i", tli, closest_backup->tli, begin_segno_str, - end_segno_str, tlinfo->tli); + end_segno_str, tlInfo->tli); - parray_append(tlinfo->keep_segments, interval); + parray_append(tlInfo->keep_segments, interval); continue; } continue; } /* Iterate over backups left */ - for (j = count; j < parray_num(tlinfo->backups); j++) + for (j = count; j < parray_num(tlInfo->backups); j++) { XLogSegNo segno = 0; xlogInterval *interval = NULL; - pgBackup *backup = parray_get(tlinfo->backups, j); + pgBackup *backup = parray_get(tlInfo->backups, j); /* * We must calculate keep_segments intervals for ARCHIVE backups @@ -2039,7 +2039,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) continue; /* no point in clogging keep_segments by backups protected by anchor_lsn */ - if (backup->start_lsn >= tlinfo->anchor_lsn) + if (backup->start_lsn >= tlInfo->anchor_lsn) continue; /* append interval to keep_segments */ @@ -2057,8 +2057,8 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) else interval->end_segno = segno; - GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size); - GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size); + GetXLogFileName(begin_segno_str, tlInfo->tli, interval->begin_segno, instance->xlog_seg_size); + GetXLogFileName(end_segno_str, tlInfo->tli, interval->end_segno, instance->xlog_seg_size); elog(LOG, "Archive backup %s to stay consistent " "protect from purge WAL interval " @@ -2066,10 +2066,10 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) backup_id_of(backup), begin_segno_str, end_segno_str, backup->tli); - if (tlinfo->keep_segments == NULL) - tlinfo->keep_segments = parray_new(); + if (tlInfo->keep_segments == NULL) + tlInfo->keep_segments = parray_new(); - parray_append(tlinfo->keep_segments, interval); + parray_append(tlInfo->keep_segments, interval); } } @@ -2081,27 +2081,27 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) for (i = 0; i < parray_num(timelineinfos); i++) { XLogSegNo anchor_segno = 0; - timelineInfo *tlinfo = parray_get(timelineinfos, i); + timelineInfo *tlInfo = parray_get(timelineinfos, i); /* * At this point invalid anchor_lsn can be only in one case: * timeline is going to be purged by regular WAL purge rules. */ - if (XLogRecPtrIsInvalid(tlinfo->anchor_lsn)) + if (XLogRecPtrIsInvalid(tlInfo->anchor_lsn)) continue; /* * anchor_lsn is located in another timeline, it means that the timeline * will be protected from purge entirely. */ - if (tlinfo->anchor_tli > 0 && tlinfo->anchor_tli != tlinfo->tli) + if (tlInfo->anchor_tli > 0 && tlInfo->anchor_tli != tlInfo->tli) continue; - GetXLogSegNo(tlinfo->anchor_lsn, anchor_segno, instance->xlog_seg_size); + GetXLogSegNo(tlInfo->anchor_lsn, anchor_segno, instance->xlog_seg_size); - for (j = 0; j < parray_num(tlinfo->xlog_filelist); j++) + for (j = 0; j < parray_num(tlInfo->xlog_filelist); j++) { - xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, j); + xlogFile *wal_file = (xlogFile *) parray_get(tlInfo->xlog_filelist, j); if (wal_file->segno >= anchor_segno) { @@ -2110,13 +2110,13 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) } /* no keep segments */ - if (!tlinfo->keep_segments) + if (!tlInfo->keep_segments) continue; /* Protect segments belonging to one of the keep invervals */ - for (k = 0; k < parray_num(tlinfo->keep_segments); k++) + for (k = 0; k < parray_num(tlInfo->keep_segments); k++) { - xlogInterval *keep_segments = (xlogInterval *) parray_get(tlinfo->keep_segments, k); + xlogInterval *keep_segments = (xlogInterval *) parray_get(tlInfo->keep_segments, k); if ((wal_file->segno >= keep_segments->begin_segno) && wal_file->segno <= keep_segments->end_segno) diff --git a/src/catchup.c b/src/catchup.c index 00752b194..39fd37d26 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -2,7 +2,7 @@ * * catchup.c: sync DB cluster * - * Copyright (c) 2021-2022, Postgres Professional + * Copyright (c) 2021-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/checkdb.c b/src/checkdb.c index 2a7d4e9eb..24c80657e 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -9,7 +9,7 @@ * instance can be logically verified using extensions * amcheck or amcheck_next. * - * Portions Copyright (c) 2019-2019, Postgres Professional + * Portions Copyright (c) 2019-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/configure.c b/src/configure.c index 964548343..59f164fa9 100644 --- a/src/configure.c +++ b/src/configure.c @@ -2,7 +2,7 @@ * * configure.c: - manage backup catalog. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/data.c b/src/data.c index 1a9616bae..d88b06b62 100644 --- a/src/data.c +++ b/src/data.c @@ -3,7 +3,7 @@ * data.c: utils to parse and backup data pages * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/datapagemap.c b/src/datapagemap.c index 7e4202a72..49ce91334 100644 --- a/src/datapagemap.c +++ b/src/datapagemap.c @@ -5,7 +5,7 @@ * * This is a fairly simple bitmap. * - * Copyright (c) 2013-2019, PostgreSQL Global Development Group + * Copyright (c) 2013-2025, PostgreSQL Global Development Group * *------------------------------------------------------------------------- */ diff --git a/src/delete.c b/src/delete.c index f48ecc95f..1c628e04f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -3,7 +3,7 @@ * delete.c: delete backup files. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/dir.c b/src/dir.c index 4b1bc2816..a5bde57f3 100644 --- a/src/dir.c +++ b/src/dir.c @@ -3,7 +3,7 @@ * dir.c: directory operation utility. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/help.c b/src/help.c index e18706a13..eacef9a48 100644 --- a/src/help.c +++ b/src/help.c @@ -2,7 +2,7 @@ * * help.c * - * Copyright (c) 2017-2021, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/init.c b/src/init.c index 837e2bad0..6afb5706c 100644 --- a/src/init.c +++ b/src/init.c @@ -3,7 +3,7 @@ * init.c: - initialize backup catalog. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/merge.c b/src/merge.c index e8f926795..3692fee8a 100644 --- a/src/merge.c +++ b/src/merge.c @@ -2,7 +2,7 @@ * * merge.c: merge FULL and incremental backups * - * Copyright (c) 2018-2022, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/parsexlog.c b/src/parsexlog.c index 7df169fbf..3dd591e52 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -5,7 +5,7 @@ * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index fa67ddff5..b72efa93a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -35,7 +35,7 @@ * which includes info about pgdata directory and connection. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ @@ -980,7 +980,13 @@ main(int argc, char *argv[]) wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: - return do_add_instance(instanceState, &instance_config); + { + PGNodeInfo nodeInfo; + pgNodeInit(&nodeInfo); + instanceState->conn = pgut_connect(dbhost, dbport, dbname, dbuser); + check_server_version(instanceState->conn, &nodeInfo); + return do_add_instance(instanceState, &instance_config); + } case DELETE_INSTANCE_CMD: return do_delete_instance(instanceState); case INIT_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ae99e0605..9ae66a3a2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ @@ -431,6 +431,8 @@ typedef struct InstanceConfig extern ConfigOption instance_options[]; extern InstanceConfig instance_config; extern time_t current_time; +extern bool no_validate; +extern IncrRestoreMode incremental_mode; typedef struct PGNodeInfo { @@ -805,9 +807,12 @@ extern pid_t my_pid; extern __thread int my_thread_num; extern int num_threads; extern bool stream_wal; +extern bool no_color; extern bool show_color; extern bool progress; +extern bool no_sync; extern bool is_archive_cmd; /* true for archive-{get,push} */ +extern time_t start_time; /* In pre-10 'replication_slot' is defined in receivelog.h */ extern char *replication_slot; #if PG_VERSION_NUM >= 100000 @@ -816,6 +821,7 @@ extern bool temp_slot; extern bool perm_slot; /* backup options */ +extern bool backup_logs; extern bool smooth_checkpoint; /* remote probackup options */ @@ -827,8 +833,15 @@ extern bool exclusive_backup; extern bool delete_wal; extern bool delete_expired; extern bool merge_expired; +extern bool force; extern bool dry_run; +/* archive push options */ +extern int batch_size; + +/* archive get options */ +extern bool no_validate_wal; + /* ===== instanceState ===== */ typedef struct InstanceState @@ -858,11 +871,18 @@ typedef struct InstanceState /* show options */ extern ShowFormat show_format; +extern bool show_archive; + +/* set backup options */ +extern int64 ttl; /* checkdb options */ +extern bool need_amcheck; extern bool heapallindexed; extern bool checkunique; +extern bool amcheck_parent; extern bool skip_block_validation; +extern bool skip_external_dirs; /* current settings */ extern pgBackup current; @@ -1225,6 +1245,7 @@ extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); +void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); extern bool parse_page(Page page, XLogRecPtr *lsn); extern int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h index 56d852537..a1b221f46 100644 --- a/src/pg_probackup_state.h +++ b/src/pg_probackup_state.h @@ -2,7 +2,7 @@ * * pg_probackup_state.h: Definitions of internal pg_probackup states * - * Portions Copyright (c) 2021, Postgres Professional + * Portions Copyright (c) 2021-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/ptrack.c b/src/ptrack.c index d27629e45..ba97088c1 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -2,7 +2,7 @@ * * ptrack.c: support functions for ptrack backups * - * Copyright (c) 2021 Postgres Professional + * Copyright (c) 2021-2025 Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/restore.c b/src/restore.c index f9310dcee..0be151a99 100644 --- a/src/restore.c +++ b/src/restore.c @@ -3,7 +3,7 @@ * restore.c: restore DB cluster and archived WAL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/show.c b/src/show.c index 810262df6..0732c6a7a 100644 --- a/src/show.c +++ b/src/show.c @@ -3,7 +3,7 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/stream.c b/src/stream.c index 77453e997..25429f9a7 100644 --- a/src/stream.c +++ b/src/stream.c @@ -2,7 +2,7 @@ * * stream.c: pg_probackup specific code for WAL streaming * - * Portions Copyright (c) 2015-2020, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ @@ -592,7 +592,7 @@ parse_tli_history_buffer(char *history, TimeLineID tli) if (curLineLen > 0) { char *ptr; - TimeLineID tli; + TimeLineID currTLI; uint32 switchpoint_hi; uint32 switchpoint_lo; int nfields; @@ -605,7 +605,7 @@ parse_tli_history_buffer(char *history, TimeLineID tli) if (*ptr == '\0' || *ptr == '#') continue; - nfields = sscanf(tempStr, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); + nfields = sscanf(tempStr, "%u\t%X/%X", &currTLI, &switchpoint_hi, &switchpoint_lo); if (nfields < 1) { @@ -615,11 +615,11 @@ parse_tli_history_buffer(char *history, TimeLineID tli) if (nfields != 3) elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a transaction log switchpoint location.", tempStr); - if (last_timeline && tli <= last_timeline->tli) + if (last_timeline && currTLI <= last_timeline->tli) elog(ERROR, "Timeline IDs must be in increasing sequence: \"%s\"", tempStr); entry = pgut_new(TimeLineHistoryEntry); - entry->tli = tli; + entry->tli = currTLI; entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; last_timeline = entry; @@ -628,7 +628,7 @@ parse_tli_history_buffer(char *history, TimeLineID tli) result = parray_new(); parray_append(result, entry); elog(VERBOSE, "parse_tli_history_buffer() found entry: tli = %X, end = %X/%X", - tli, switchpoint_hi, switchpoint_lo); + currTLI, switchpoint_hi, switchpoint_lo); /* we ignore the remainder of each line */ } diff --git a/src/util.c b/src/util.c index 3c0a33453..019d20644 100644 --- a/src/util.c +++ b/src/util.c @@ -3,7 +3,7 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/configuration.c b/src/utils/configuration.c index f049aa1be..47497850f 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -3,7 +3,7 @@ * configuration.c: - function implementations to work with pg_probackup * configurations. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 59da29bd5..da86b9db0 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -3,7 +3,7 @@ * configuration.h: - prototypes of functions and structures for * configuration. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/file.c b/src/utils/file.c index fa08939f5..b49c97d0c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2499,7 +2499,7 @@ fio_send_pages_impl(int out, char* buf) int fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) { - fio_header hdr; + fio_header header; int exit_code = SEND_OK; char *in_buf = pgut_malloc(CHUNK_SIZE); /* buffer for compressed data */ char *out_buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer for decompressed data */ @@ -2507,13 +2507,13 @@ fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) /* decompressor */ z_stream *strm = NULL; - hdr.cop = FIO_SEND_FILE; - hdr.size = path_len; + header.cop = FIO_SEND_FILE; + header.size = path_len; // elog(VERBOSE, "Thread [%d]: Attempting to open remote compressed WAL file '%s'", // thread_num, from_fullpath); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &header, sizeof(header)), sizeof(header)); IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); for (;;) diff --git a/src/utils/json.c b/src/utils/json.c index 2c8e0fe9b..1bcb4e644 100644 --- a/src/utils/json.c +++ b/src/utils/json.c @@ -2,7 +2,7 @@ * * json.c: - make json document. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/json.h b/src/utils/json.h index f80832e69..f635e1f3c 100644 --- a/src/utils/json.h +++ b/src/utils/json.h @@ -2,7 +2,7 @@ * * json.h: - prototypes of json output functions. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 7ea41f74e..ade57e529 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -2,7 +2,7 @@ * * logger.c: - log events into log file or stderr. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.h b/src/utils/logger.h index adc5061e0..9e2cb958f 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -2,7 +2,7 @@ * * logger.h: - prototypes of logger functions. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 9559fa644..df09dbdc6 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -3,7 +3,7 @@ * pgut.c * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 1b7b7864c..6fea4d022 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/remote.h b/src/utils/remote.h index dc98644ab..582210142 100644 --- a/src/utils/remote.h +++ b/src/utils/remote.h @@ -2,7 +2,7 @@ * * remote.h: - prototypes of remote functions. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/thread.c b/src/utils/thread.c index 1c469bd29..4127701f0 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -2,7 +2,7 @@ * * thread.c: - multi-platform pthread implementations. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/thread.h b/src/utils/thread.h index 2eaa5fb45..d79e2d8d0 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -2,7 +2,7 @@ * * thread.h: - multi-platform pthread implementations. * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/validate.c b/src/validate.c index 0887b2e7a..3bff3f756 100644 --- a/src/validate.c +++ b/src/validate.c @@ -3,7 +3,7 @@ * validate.c: validate backup files. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2025, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/tests/archive_test.py b/tests/archive_test.py index 00fd1f592..9bd37aa55 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -8,7 +8,6 @@ import subprocess from sys import exit from time import sleep -from distutils.dir_util import copy_tree class ArchiveTest(ProbackupTest, unittest.TestCase): @@ -1243,10 +1242,6 @@ def test_archive_catalog(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) # FULL backup replica diff --git a/tests/backup_test.py b/tests/backup_test.py index dc60228b5..5ec6f1c6e 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -4,7 +4,6 @@ from time import sleep, time from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException import shutil -from distutils.dir_util import copy_tree from testgres import ProcessType, QueryException import subprocess @@ -2330,14 +2329,13 @@ def test_backup_with_less_privileges_role(self): # bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] # gdb_checkpointer = self.gdb_attach(bgwriter_pid) - copy_tree( - os.path.join(backup_dir, 'wal', 'node'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) - # self.switch_wal_segment(node) - # self.switch_wal_segment(node) + # make sure replica will archive wal segment with backup start point + lsn = self.switch_wal_segment(node, and_tx=True) + replica.poll_query_until(f"select pg_last_wal_replay_lsn() >= '{lsn}'") + replica.execute('CHECKPOINT') + replica.poll_query_until(f"select redo_lsn >= '{lsn}' from pg_control_checkpoint()") self.backup_node( backup_dir, 'replica', replica, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 27d982856..2420be462 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1701,29 +1701,26 @@ def version_to_num(self, version): num = num * 100 + int(re.sub(r"[^\d]", "", part)) return num - def switch_wal_segment(self, node): + def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): """ - Execute pg_switch_wal/xlog() in given node + Execute pg_switch_wal() in given node Args: node: an instance of PostgresNode or NodeConnection class """ if isinstance(node, testgres.PostgresNode): - if self.version_to_num( - node.safe_psql('postgres', 'show server_version').decode('utf-8') - ) >= self.version_to_num('10.0'): - node.safe_psql('postgres', 'select pg_switch_wal()') - else: - node.safe_psql('postgres', 'select pg_switch_xlog()') + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + con.execute('select pg_switch_wal()') + lsn = con.execute('select pg_switch_wal()')[0][0] else: - if self.version_to_num( - node.execute('show server_version')[0][0] - ) >= self.version_to_num('10.0'): - node.execute('select pg_switch_wal()') - else: - node.execute('select pg_switch_xlog()') + node.execute('select pg_switch_wal()') + lsn = node.execute('select pg_switch_wal()')[0][0] - sleep(1) + if sleep_seconds > 0: + sleep(sleep_seconds) + return lsn def wait_until_replica_catch_with_master(self, master, replica): diff --git a/tests/replica_test.py b/tests/replica_test.py index 17fc5a823..e96e0dfa7 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta import subprocess import time -from distutils.dir_util import copy_tree from testgres import ProcessType from time import sleep @@ -718,10 +717,6 @@ def test_replica_stop_lsn_null_offset_next_record(self): self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) self.switch_wal_segment(master) @@ -980,10 +975,6 @@ def test_replica_toast(self): self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) self.switch_wal_segment(master) diff --git a/tests/requirements.txt b/tests/requirements.txt index e2ac18bea..31e01aeb5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,7 +5,7 @@ # git+https://github.com/postgrespro/testgres.git@ # 3. From a local directory # /path/to/local/directory/testgres -git+https://github.com/postgrespro/testgres.git@archive-command-exec#egg=testgres-pg_probackup2&subdirectory=testgres/plugins/pg_probackup2 +testgres==1.8.5 allure-pytest deprecation pexpect diff --git a/tests/retention_test.py b/tests/retention_test.py index 88432a00f..cf422fe04 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from time import sleep -from distutils.dir_util import copy_tree class RetentionTest(ProbackupTest, unittest.TestCase): diff --git a/version.sh b/version.sh new file mode 100755 index 000000000..0e271bcd4 --- /dev/null +++ b/version.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# Output the program version based on the state of the repository (source code +# and tags). +# Tags must be in the form of +# x.y.z, e.g. 3.1.0 +# or +# x.y.z-, e.g. 3.1.0-beta2 +# If the tag consists only of the version number (i.e. it doesn't contain a +# prerelease part) and this number is equal to the version in the header file +# then the version is considered a release and no additional version data is +# appended to it by default (but can be forced by "-p" and "-r" command line +# arguments). Otherwise, provided Git is available, the prerelease part and Git +# revision are automatically added to the version. + +cd `dirname "$0"` + +while getopts p:r:sv: opt; do + case $opt in + p) prerelease=$OPTARG;; + r) revision=$OPTARG;; + s) ID=semver;; + v) version=$OPTARG;; + esac +done + +if [ -z "$ID" ]; then + . /etc/os-release +fi +case $ID in + altlinux | astra | debian | ubuntu) + # The only scheme that properly sorts metadata and prerelease fields is + # when the both are specified after a '~' + presep='~'; metasep='~';; + centos | opensuse-leap | redos) + presep='~'; metasep=^;; + *) # semver + presep=-; metasep=+ +esac + +if [ -z "$version" ]; then + version=`grep '#define PROGRAM_VERSION' src/pg_probackup.h | cut -f 2 | tr -d '"'` +fi + +if which git >/dev/null 2>&1; then + tag=`git describe --tags 2> /dev/null` + # Shallow cloned repository may not have tags + if [ -z "$prerelease" -a "$tag" ]; then + f1=`cut -d - -f 1 <<< $tag` + f2=`cut -d - -f 2 <<< $tag` + # Append the prerelease part only if the tag refers to the current version + # Assume that the prerelease part contains letters + if [ $f1 = $version ] && expr "$f2" : "[1-9]*[a-zA-Z]" 1>/dev/null; then + prerelease="$f2" + fi + fi + if [ -z "$revision" ]; then + revision=g`git rev-parse --short HEAD` + fi +fi + +out=$version${prerelease:+$presep$prerelease} +if [ "$tag" != $version -a "$revision" ]; then + out=$out$metasep`date +%Y%m%d`$revision +fi + +echo $out