Skip to content

Commit 3fe3511

Browse files
simonat2ndQuadrantsimonat2ndQuadrant
committed
Generic Messages for Logical Decoding
API and mechanism to allow generic messages to be inserted into WAL that are intended to be read by logical decoding plugins. This commit adds an optional new callback to the logical decoding API. Messages are either text or bytea. Messages can be transactional, or not, and are identified by a prefix to allow multiple concurrent decoding plugins. (Not to be confused with Generic WAL records, which are intended to allow crash recovery of extensible objects.) Author: Petr Jelinek and Andres Freund Reviewers: Artur Zakirov, Tomas Vondra, Simon Riggs Discussion: 5685F999.6010202@2ndquadrant.com
1 parent 989be08 commit 3fe3511

File tree

27 files changed

+693
-33
lines changed

27 files changed

+693
-33
lines changed

‎contrib/test_decoding/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ submake-test_decoding:
3838
$(MAKE) -C $(top_builddir)/contrib/test_decoding
3939

4040
REGRESSCHECKS=ddl xact rewrite toast permissions decoding_in_xact \
41-
decoding_into_rel binary prepared replorigin time
41+
decoding_into_rel binary prepared replorigin time messages
4242

4343
regresscheck: | submake-regress submake-test_decoding temp-install
4444
$(MKDIR_P) regression_output

‎contrib/test_decoding/expected/ddl.out‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -220,11 +220,17 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'inc
220220
(7 rows)
221221

222222
/*
223-
* check that disk spooling works
223+
* check that disk spooling works (also for logical messages)
224224
*/
225225
BEGIN;
226226
CREATE TABLE tr_etoomuch (id serial primary key, data int);
227227
INSERT INTO tr_etoomuch(data) SELECT g.i FROM generate_series(1, 10234) g(i);
228+
SELECT 'tx logical msg' FROM pg_logical_emit_message(true, 'test', 'tx logical msg');
229+
?column?
230+
----------------
231+
tx logical msg
232+
(1 row)
233+
228234
DELETE FROM tr_etoomuch WHERE id < 5000;
229235
UPDATE tr_etoomuch SET data = - data WHERE id > 5000;
230236
COMMIT;
@@ -233,12 +239,13 @@ SELECT count(*), min(data), max(data)
233239
FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1')
234240
GROUP BY substring(data, 1, 24)
235241
ORDER BY 1,2;
236-
count | min | max
237-
-------+-------------------------------------------------+------------------------------------------------------------------------
238-
1 | BEGIN | BEGIN
239-
1 | COMMIT | COMMIT
240-
20467 | table public.tr_etoomuch: DELETE: id[integer]:1 | table public.tr_etoomuch: UPDATE: id[integer]:9999 data[integer]:-9999
241-
(3 rows)
242+
count | min | max
243+
-------+-----------------------------------------------------------------------+------------------------------------------------------------------------
244+
1 | BEGIN | BEGIN
245+
1 | COMMIT | COMMIT
246+
1 | message: transactional: 1 prefix: test, sz: 14 content:tx logical msg | message: transactional: 1 prefix: test, sz: 14 content:tx logical msg
247+
20467 | table public.tr_etoomuch: DELETE: id[integer]:1 | table public.tr_etoomuch: UPDATE: id[integer]:9999 data[integer]:-9999
248+
(4 rows)
242249

243250
-- check updates of primary keys work correctly
244251
BEGIN;

‎contrib/test_decoding/expected/messages.out‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
-- predictability
2+
SET synchronous_commit = on;
3+
SET client_encoding = 'utf8';
4+
SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding');
5+
?column?
6+
----------
7+
init
8+
(1 row)
9+
10+
SELECT 'msg1' FROM pg_logical_emit_message(true, 'test', 'msg1');
11+
?column?
12+
----------
13+
msg1
14+
(1 row)
15+
16+
SELECT 'msg2' FROM pg_logical_emit_message(false, 'test', 'msg2');
17+
?column?
18+
----------
19+
msg2
20+
(1 row)
21+
22+
BEGIN;
23+
SELECT 'msg3' FROM pg_logical_emit_message(true, 'test', 'msg3');
24+
?column?
25+
----------
26+
msg3
27+
(1 row)
28+
29+
SELECT 'msg4' FROM pg_logical_emit_message(false, 'test', 'msg4');
30+
?column?
31+
----------
32+
msg4
33+
(1 row)
34+
35+
ROLLBACK;
36+
BEGIN;
37+
SELECT 'msg5' FROM pg_logical_emit_message(true, 'test', 'msg5');
38+
?column?
39+
----------
40+
msg5
41+
(1 row)
42+
43+
SELECT 'msg6' FROM pg_logical_emit_message(false, 'test', 'msg6');
44+
?column?
45+
----------
46+
msg6
47+
(1 row)
48+
49+
SELECT 'msg7' FROM pg_logical_emit_message(true, 'test', 'msg7');
50+
?column?
51+
----------
52+
msg7
53+
(1 row)
54+
55+
COMMIT;
56+
SELECT 'žluťoučký kůň' FROM pg_logical_emit_message(true, 'test', 'žluťoučký kůň');
57+
?column?
58+
---------------
59+
žluťoučký kůň
60+
(1 row)
61+
62+
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'force-binary', '0', 'skip-empty-xacts', '1');
63+
data
64+
----------------------------------------------------------------------
65+
message: transactional: 1 prefix: test, sz: 4 content:msg1
66+
message: transactional: 0 prefix: test, sz: 4 content:msg2
67+
message: transactional: 0 prefix: test, sz: 4 content:msg4
68+
message: transactional: 0 prefix: test, sz: 4 content:msg6
69+
message: transactional: 1 prefix: test, sz: 4 content:msg5
70+
message: transactional: 1 prefix: test, sz: 4 content:msg7
71+
message: transactional: 1 prefix: test, sz: 19 content:žluťoučký kůň
72+
(7 rows)
73+
74+
SELECT 'init' FROM pg_drop_replication_slot('regression_slot');
75+
?column?
76+
----------
77+
init
78+
(1 row)
79+

‎contrib/test_decoding/sql/ddl.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,11 +108,12 @@ DELETE FROM tr_pkey;
108108
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
109109

110110
/*
111-
* check that disk spooling works
111+
* check that disk spooling works (also for logical messages)
112112
*/
113113
BEGIN;
114114
CREATE TABLE tr_etoomuch (id serial primary key, data int);
115115
INSERT INTO tr_etoomuch(data) SELECT g.i FROM generate_series(1, 10234) g(i);
116+
SELECT 'tx logical msg' FROM pg_logical_emit_message(true, 'test', 'tx logical msg');
116117
DELETE FROM tr_etoomuch WHERE id < 5000;
117118
UPDATE tr_etoomuch SET data = - data WHERE id > 5000;
118119
COMMIT;

‎contrib/test_decoding/sql/messages.sql‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
-- predictability
2+
SET synchronous_commit = on;
3+
SET client_encoding = 'utf8';
4+
5+
SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding');
6+
7+
SELECT 'msg1' FROM pg_logical_emit_message(true, 'test', 'msg1');
8+
SELECT 'msg2' FROM pg_logical_emit_message(false, 'test', 'msg2');
9+
10+
BEGIN;
11+
SELECT 'msg3' FROM pg_logical_emit_message(true, 'test', 'msg3');
12+
SELECT 'msg4' FROM pg_logical_emit_message(false, 'test', 'msg4');
13+
ROLLBACK;
14+
15+
BEGIN;
16+
SELECT 'msg5' FROM pg_logical_emit_message(true, 'test', 'msg5');
17+
SELECT 'msg6' FROM pg_logical_emit_message(false, 'test', 'msg6');
18+
SELECT 'msg7' FROM pg_logical_emit_message(true, 'test', 'msg7');
19+
COMMIT;
20+
21+
SELECT 'žluťoučký kůň' FROM pg_logical_emit_message(true, 'test', 'žluťoučký kůň');
22+
23+
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'force-binary', '0', 'skip-empty-xacts', '1');
24+
25+
SELECT 'init' FROM pg_drop_replication_slot('regression_slot');

‎contrib/test_decoding/test_decoding.c‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include "replication/output_plugin.h"
2323
#include "replication/logical.h"
24+
#include "replication/message.h"
2425
#include "replication/origin.h"
2526

2627
#include "utils/builtins.h"
@@ -63,6 +64,10 @@ static void pg_decode_change(LogicalDecodingContext *ctx,
6364
ReorderBufferChange *change);
6465
static bool pg_decode_filter(LogicalDecodingContext *ctx,
6566
RepOriginId origin_id);
67+
static void pg_decode_message(LogicalDecodingContext *ctx,
68+
ReorderBufferTXN *txn, XLogRecPtr message_lsn,
69+
bool transactional, const char *prefix,
70+
Size sz, const char *message);
6671

6772
void
6873
_PG_init(void)
@@ -82,6 +87,7 @@ _PG_output_plugin_init(OutputPluginCallbacks *cb)
8287
cb->commit_cb = pg_decode_commit_txn;
8388
cb->filter_by_origin_cb = pg_decode_filter;
8489
cb->shutdown_cb = pg_decode_shutdown;
90+
cb->message_cb = pg_decode_message;
8591
}
8692

8793

@@ -471,3 +477,15 @@ pg_decode_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
471477

472478
OutputPluginWrite(ctx, true);
473479
}
480+
481+
static void
482+
pg_decode_message(LogicalDecodingContext *ctx,
483+
ReorderBufferTXN *txn, XLogRecPtr lsn, bool transactional,
484+
const char *prefix, Size sz, const char *message)
485+
{
486+
OutputPluginPrepareWrite(ctx, true);
487+
appendStringInfo(ctx->out, "message: transactional: %d prefix: %s, sz: %zu content:",
488+
transactional, prefix, sz);
489+
appendBinaryStringInfo(ctx->out, message, sz);
490+
OutputPluginWrite(ctx, true);
491+
}

‎doc/src/sgml/func.sgml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18255,6 +18255,51 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup());
1825518255
</entry>
1825618256
</row>
1825718257

18258+
<row>
18259+
<entry id="pg-logical-emit-message-text">
18260+
<indexterm>
18261+
<primary>pg_logical_emit_message</primary>
18262+
</indexterm>
18263+
<literal><function>pg_logical_emit_message(<parameter>transactional</parameter> <type>bool</type>, <parameter>prefix</parameter> <type>text</type>, <parameter>content</parameter> <type>text</type>)</function></literal>
18264+
</entry>
18265+
<entry>
18266+
void
18267+
</entry>
18268+
<entry>
18269+
Emit text logical decoding message. This can be used to pass generic
18270+
messages to logical decoding plugins through WAL. The parameter
18271+
<parameter>transactional</parameter> specifies if the message should
18272+
be part of current transaction or if it should be written immediately
18273+
and decoded as soon as the logical decoding reads the record. The
18274+
<parameter>prefix</parameter> is textual prefix used by the logical
18275+
decoding plugins to easily recognize interesting messages for them.
18276+
The <parameter>content</parameter> is the text of the message.
18277+
</entry>
18278+
</row>
18279+
18280+
<row>
18281+
<entry id="pg-logical-emit-message-bytea">
18282+
<indexterm>
18283+
<primary>>pg_logical_emit_message</primary>
18284+
</indexterm>
18285+
<literal><function>>pg_logical_emit_message(<parameter>transactional</parameter> <type>bool</type>, <parameter>prefix</parameter> <type>text</type>, <parameter>content</parameter> <type>bytea</type>)</function></literal>
18286+
</entry>
18287+
<entry>
18288+
void
18289+
</entry>
18290+
<entry>
18291+
Emit binary logical decoding message. This can be used to pass generic
18292+
messages to logical decoding plugins through WAL. The parameter
18293+
<parameter>transactional</parameter> specifies if the message should
18294+
be part of current transaction or if it should be written immediately
18295+
and decoded as soon as the logical decoding reads the record. The
18296+
<parameter>prefix</parameter> is textual prefix used by the logical
18297+
decoding plugins to easily recognize interesting messages for them.
18298+
The <parameter>content</parameter> is the binary content of the
18299+
message.
18300+
</entry>
18301+
</row>
18302+
1825818303
</tbody>
1825918304
</tgroup>
1826018305
</table>

‎doc/src/sgml/logicaldecoding.sgml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,7 @@ typedef struct OutputPluginCallbacks
363363
LogicalDecodeBeginCB begin_cb;
364364
LogicalDecodeChangeCB change_cb;
365365
LogicalDecodeCommitCB commit_cb;
366+
LogicalDecodeMessageCB message_cb;
366367
LogicalDecodeFilterByOriginCB filter_by_origin_cb;
367368
LogicalDecodeShutdownCB shutdown_cb;
368369
} OutputPluginCallbacks;
@@ -602,6 +603,43 @@ typedef bool (*LogicalDecodeFilterByOriginCB) (
602603
more efficient.
603604
</para>
604605
</sect3>
606+
607+
<sect3 id="logicaldecoding-output-plugin-message">
608+
<title>Generic Message Callback</title>
609+
610+
<para>
611+
The optional <function>message_cb</function> callback is called whenever
612+
a logical decoding message has been decoded.
613+
<programlisting>
614+
typedef void (*LogicalDecodeMessageCB) (
615+
struct LogicalDecodingContext *,
616+
ReorderBufferTXN *txn,
617+
XLogRecPtr message_lsn,
618+
bool transactional,
619+
const char *prefix,
620+
Size message_size,
621+
const char *message
622+
);
623+
</programlisting>
624+
The <parameter>txn</parameter> parameter contains meta information about
625+
the transaction, like the time stamp at which it has been committed and
626+
its XID. Note however that it can be NULL when the message is
627+
non-transactional and the XID was not assigned yet in the transaction
628+
which logged the message. The <parameter>lsn</parameter> has WAL
629+
position of the message. The <parameter>transactional</parameter> says
630+
if the message was sent as transactional or not.
631+
The <parameter>prefix</parameter> is arbitrary null-terminated prefix
632+
which can be used for identifying interesting messages for the current
633+
plugin. And finally the <parameter>message</parameter> parameter holds
634+
the actual message of <parameter>message_size</parameter> size.
635+
</para>
636+
<para>
637+
Extra care should be taken to ensure that the prefix the output plugin
638+
considers interesting is unique. Using name of the extension or the
639+
output plugin itself is often a good choice.
640+
</para>
641+
</sect3>
642+
605643
</sect2>
606644

607645
<sect2 id="logicaldecoding-output-plugin-output">

‎src/backend/access/rmgrdesc/Makefile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ top_builddir = ../../../..
99
include $(top_builddir)/src/Makefile.global
1010

1111
OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
12-
gindesc.o gistdesc.o hashdesc.o heapdesc.o mxactdesc.o nbtdesc.o \
13-
relmapdesc.o replorigindesc.o seqdesc.o smgrdesc.o spgdesc.o \
14-
standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
12+
gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
13+
mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
14+
smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
1515

1616
include $(top_srcdir)/src/backend/common.mk

‎src/backend/access/rmgrdesc/logicalmsgdesc.c‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*-------------------------------------------------------------------------
2+
*
3+
* logicalmsgdesc.c
4+
* rmgr descriptor routines for replication/logical/message.c
5+
*
6+
* Portions Copyright (c) 2015-2016, PostgreSQL Global Development Group
7+
*
8+
*
9+
* IDENTIFICATION
10+
* src/backend/access/rmgrdesc/logicalmsgdesc.c
11+
*
12+
*-------------------------------------------------------------------------
13+
*/
14+
#include "postgres.h"
15+
16+
#include "replication/message.h"
17+
18+
void
19+
logicalmsg_desc(StringInfo buf, XLogReaderState *record)
20+
{
21+
char *rec = XLogRecGetData(record);
22+
uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
23+
24+
if (info == XLOG_LOGICAL_MESSAGE)
25+
{
26+
xl_logical_message *xlrec = (xl_logical_message *) rec;
27+
28+
appendStringInfo(buf, "%s message size %zu bytes",
29+
xlrec->transactional ? "transactional" : "nontransactional",
30+
xlrec->message_size);
31+
}
32+
}
33+
34+
const char *
35+
logicalmsg_identify(uint8 info)
36+
{
37+
if ((info & ~XLR_INFO_MASK) == XLOG_LOGICAL_MESSAGE)
38+
return "MESSAGE";
39+
40+
return NULL;
41+
}

0 commit comments

Comments
 (0)