Skip to content

Commit dfda6eb

Browse files
hlinnakahlinnaka
committed
Don't waste the last segment of each 4GB logical log file.
The comments claimed that wasting the last segment made it easier to do calculations with XLogRecPtrs, because you don't have problems representing last-byte-position-plus-1 that way. In my experience, however, it only made things more complicated, because the there was two ways to represent the boundary at the beginning of a logical log file: logid = n+1 and xrecoff = 0, or as xlogid = n and xrecoff = 4GB - XLOG_SEG_SIZE. Some functions were picky about which representation was used. Also, use a 64-bit segment number instead of the log/seg combination, to point to a certain WAL segment. We assume that all platforms have a working 64-bit integer type nowadays. This is an incompatible change in WAL format, so bumping WAL version number.
1 parent 47c7365 commit dfda6eb

File tree

12 files changed

+415
-556
lines changed

12 files changed

+415
-556
lines changed

‎src/backend/access/transam/xlog.c‎

Lines changed: 245 additions & 307 deletions
Large diffs are not rendered by default.

‎src/backend/access/transam/xlogfuncs.c‎

Lines changed: 16 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -271,8 +271,7 @@ pg_xlogfile_name_offset(PG_FUNCTION_ARGS)
271271
char *locationstr;
272272
unsigned int uxlogid;
273273
unsigned int uxrecoff;
274-
uint32 xlogid;
275-
uint32 xlogseg;
274+
XLogSegNo xlogsegno;
276275
uint32 xrecoff;
277276
XLogRecPtr locationpoint;
278277
char xlogfilename[MAXFNAMELEN];
@@ -319,16 +318,16 @@ pg_xlogfile_name_offset(PG_FUNCTION_ARGS)
319318
/*
320319
* xlogfilename
321320
*/
322-
XLByteToPrevSeg(locationpoint, xlogid, xlogseg);
323-
XLogFileName(xlogfilename, ThisTimeLineID, xlogid, xlogseg);
321+
XLByteToPrevSeg(locationpoint, xlogsegno);
322+
XLogFileName(xlogfilename, ThisTimeLineID, xlogsegno);
324323

325324
values[0] = CStringGetTextDatum(xlogfilename);
326325
isnull[0] = false;
327326

328327
/*
329328
* offset
330329
*/
331-
xrecoff = locationpoint.xrecoff - xlogseg * XLogSegSize;
330+
xrecoff = locationpoint.xrecoff % XLogSegSize;
332331

333332
values[1] = UInt32GetDatum(xrecoff);
334333
isnull[1] = false;
@@ -354,8 +353,7 @@ pg_xlogfile_name(PG_FUNCTION_ARGS)
354353
char *locationstr;
355354
unsigned int uxlogid;
356355
unsigned int uxrecoff;
357-
uint32 xlogid;
358-
uint32 xlogseg;
356+
XLogSegNo xlogsegno;
359357
XLogRecPtr locationpoint;
360358
char xlogfilename[MAXFNAMELEN];
361359

@@ -378,8 +376,8 @@ pg_xlogfile_name(PG_FUNCTION_ARGS)
378376
locationpoint.xlogid = uxlogid;
379377
locationpoint.xrecoff = uxrecoff;
380378

381-
XLByteToPrevSeg(locationpoint, xlogid, xlogseg);
382-
XLogFileName(xlogfilename, ThisTimeLineID, xlogid, xlogseg);
379+
XLByteToPrevSeg(locationpoint, xlogsegno);
380+
XLogFileName(xlogfilename, ThisTimeLineID, xlogsegno);
383381

384382
PG_RETURN_TEXT_P(cstring_to_text(xlogfilename));
385383
}
@@ -514,6 +512,8 @@ pg_xlog_location_diff(PG_FUNCTION_ARGS)
514512
XLogRecPtr loc1,
515513
loc2;
516514
Numeric result;
515+
uint64 bytes1,
516+
bytes2;
517517

518518
/*
519519
* Read and parse input
@@ -533,33 +533,17 @@ pg_xlog_location_diff(PG_FUNCTION_ARGS)
533533
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
534534
errmsg("could not parse transaction log location \"%s\"", str2)));
535535

536-
/*
537-
* Sanity check
538-
*/
539-
if (loc1.xrecoff > XLogFileSize)
540-
ereport(ERROR,
541-
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
542-
errmsg("xrecoff \"%X\" is out of valid range, 0..%X", loc1.xrecoff, XLogFileSize)));
543-
if (loc2.xrecoff > XLogFileSize)
544-
ereport(ERROR,
545-
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
546-
errmsg("xrecoff \"%X\" is out of valid range, 0..%X", loc2.xrecoff, XLogFileSize)));
536+
bytes1 = (((uint64)loc1.xlogid) << 32L) + loc1.xrecoff;
537+
bytes2 = (((uint64)loc2.xlogid) << 32L) + loc2.xrecoff;
547538

548539
/*
549-
* result = XLogFileSize * (xlogid1 - xlogid2) + xrecoff1 - xrecoff2
540+
* result = bytes1 - bytes2.
541+
*
542+
* XXX: this won't handle values higher than 2^63 correctly.
550543
*/
551544
result = DatumGetNumeric(DirectFunctionCall2(numeric_sub,
552-
DirectFunctionCall1(int8_numeric, Int64GetDatum((int64) loc1.xlogid)),
553-
DirectFunctionCall1(int8_numeric, Int64GetDatum((int64) loc2.xlogid))));
554-
result = DatumGetNumeric(DirectFunctionCall2(numeric_mul,
555-
DirectFunctionCall1(int8_numeric, Int64GetDatum((int64) XLogFileSize)),
556-
NumericGetDatum(result)));
557-
result = DatumGetNumeric(DirectFunctionCall2(numeric_add,
558-
NumericGetDatum(result),
559-
DirectFunctionCall1(int8_numeric, Int64GetDatum((int64) loc1.xrecoff))));
560-
result = DatumGetNumeric(DirectFunctionCall2(numeric_sub,
561-
NumericGetDatum(result),
562-
DirectFunctionCall1(int8_numeric, Int64GetDatum((int64) loc2.xrecoff))));
545+
DirectFunctionCall1(int8_numeric, Int64GetDatum((int64) bytes1)),
546+
DirectFunctionCall1(int8_numeric, Int64GetDatum((int64) bytes2))));
563547

564548
PG_RETURN_NUMERIC(result);
565549
}

‎src/backend/postmaster/checkpointer.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -779,7 +779,7 @@ IsCheckpointOnSchedule(double progress)
779779
{
780780
recptr = GetInsertRecPtr();
781781
elapsed_xlogs =
782-
(((double) (int32) (recptr.xlogid - ckpt_start_recptr.xlogid)) * XLogSegsPerFile +
782+
(((double) ((uint64) (recptr.xlogid - ckpt_start_recptr.xlogid) << 32L)) +
783783
((double) recptr.xrecoff - (double) ckpt_start_recptr.xrecoff) / XLogSegSize) /
784784
CheckPointSegments;
785785

‎src/backend/replication/basebackup.c‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -221,10 +221,8 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
221221
* We've left the last tar file "open", so we can now append the
222222
* required WAL files to it.
223223
*/
224-
uint32 logid,
225-
logseg;
226-
uint32 endlogid,
227-
endlogseg;
224+
XLogSegNo logsegno;
225+
XLogSegNo endlogsegno;
228226
struct stat statbuf;
229227

230228
MemSet(&statbuf, 0, sizeof(statbuf));
@@ -236,16 +234,16 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
236234
statbuf.st_size = XLogSegSize;
237235
statbuf.st_mtime = time(NULL);
238236

239-
XLByteToSeg(startptr, logid, logseg);
240-
XLByteToPrevSeg(endptr, endlogid, endlogseg);
237+
XLByteToSeg(startptr, logsegno);
238+
XLByteToPrevSeg(endptr, endlogsegno);
241239

242240
while (true)
243241
{
244242
/* Send another xlog segment */
245243
char fn[MAXPGPATH];
246244
int i;
247245

248-
XLogFilePath(fn, ThisTimeLineID, logid, logseg);
246+
XLogFilePath(fn, ThisTimeLineID, logsegno);
249247
_tarWriteHeader(fn, NULL, &statbuf);
250248

251249
/* Send the actual WAL file contents, block-by-block */
@@ -254,8 +252,7 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
254252
char buf[TAR_SEND_SIZE];
255253
XLogRecPtr ptr;
256254

257-
ptr.xlogid = logid;
258-
ptr.xrecoff = logseg * XLogSegSize + TAR_SEND_SIZE * i;
255+
XLogSegNoOffsetToRecPtr(logsegno, TAR_SEND_SIZE * i, ptr);
259256

260257
/*
261258
* Some old compilers, e.g. gcc 2.95.3/x86, think that passing
@@ -277,11 +274,10 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
277274

278275

279276
/* Advance to the next WAL file */
280-
NextLogSeg(logid, logseg);
277+
logsegno++;
281278

282279
/* Have we reached our stop position yet? */
283-
if (logid > endlogid ||
284-
(logid == endlogid && logseg > endlogseg))
280+
if (logsegno > endlogsegno)
285281
break;
286282
}
287283

‎src/backend/replication/walreceiver.c‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,12 @@ walrcv_disconnect_type walrcv_disconnect = NULL;
6969

7070
/*
7171
* These variables are used similarly to openLogFile/Id/Seg/Off,
72-
* but for walreceiver to write the XLOG.
72+
* but for walreceiver to write the XLOG. recvFileTLI is the TimeLineID
73+
* corresponding the filename of recvFile, used for error messages.
7374
*/
7475
static int recvFile = -1;
75-
static uint32 recvId = 0;
76-
static uint32 recvSeg = 0;
76+
static TimeLineID recvFileTLI = -1;
77+
static XLogSegNo recvSegNo = 0;
7778
static uint32 recvOff = 0;
7879

7980
/*
@@ -481,7 +482,7 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr)
481482
{
482483
int segbytes;
483484

484-
if (recvFile < 0 || !XLByteInSeg(recptr, recvId, recvSeg))
485+
if (recvFile < 0 || !XLByteInSeg(recptr, recvSegNo))
485486
{
486487
bool use_existent;
487488

@@ -501,15 +502,16 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr)
501502
if (close(recvFile) != 0)
502503
ereport(PANIC,
503504
(errcode_for_file_access(),
504-
errmsg("could not close log file %u, segment %u: %m",
505-
recvId, recvSeg)));
505+
errmsg("could not close log segment %s: %m",
506+
XLogFileNameP(recvFileTLI, recvSegNo))));
506507
}
507508
recvFile = -1;
508509

509510
/* Create/use new log file */
510-
XLByteToSeg(recptr, recvId, recvSeg);
511+
XLByteToSeg(recptr, recvSegNo);
511512
use_existent = true;
512-
recvFile = XLogFileInit(recvId, recvSeg, &use_existent, true);
513+
recvFile = XLogFileInit(recvSegNo, &use_existent, true);
514+
recvFileTLI = ThisTimeLineID;
513515
recvOff = 0;
514516
}
515517

@@ -527,9 +529,9 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr)
527529
if (lseek(recvFile, (off_t) startoff, SEEK_SET) < 0)
528530
ereport(PANIC,
529531
(errcode_for_file_access(),
530-
errmsg("could not seek in log file %u, "
531-
"segment %u to offset %u: %m",
532-
recvId, recvSeg, startoff)));
532+
errmsg("could not seek in log segment %s, to offset %u: %m",
533+
XLogFileNameP(recvFileTLI, recvSegNo),
534+
startoff)));
533535
recvOff = startoff;
534536
}
535537

@@ -544,9 +546,9 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr)
544546
errno = ENOSPC;
545547
ereport(PANIC,
546548
(errcode_for_file_access(),
547-
errmsg("could not write to log file %u, segment %u "
549+
errmsg("could not write to log segment %s "
548550
"at offset %u, length %lu: %m",
549-
recvId, recvSeg,
551+
XLogFileNameP(recvFileTLI, recvSegNo),
550552
recvOff, (unsigned long) segbytes)));
551553
}
552554

@@ -575,7 +577,7 @@ XLogWalRcvFlush(bool dying)
575577
/* use volatile pointer to prevent code rearrangement */
576578
volatile WalRcvData *walrcv = WalRcv;
577579

578-
issue_xlog_fsync(recvFile, recvId, recvSeg);
580+
issue_xlog_fsync(recvFile, recvSegNo);
579581

580582
LogstreamResult.Flush = LogstreamResult.Write;
581583

0 commit comments

Comments
 (0)