Only allow returning string types or bytea from json_serialize

These are documented to be the allowed types for the RETURNING clause,
but the restriction was not being enforced, which caused a segfault if
another type was specified. Add some testing for this.

Per report from a.kozhemyakin

Backpatch to release 15.

Author: adunstan

src/backend/parser/parse_expr.c

@@ -4574,7 +4574,24 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
 	JsonReturning *returning;
 
 	if (expr->output)
+	{
 		returning = transformJsonOutput(pstate, expr->output, true);
+
+		if (returning->typid != BYTEAOID)
+		{
+			char		typcategory;
+			bool		typispreferred;
+
+			get_type_category_preferred(returning->typid, &typcategory,
+										&typispreferred);
+			if (typcategory != TYPCATEGORY_STRING)
+				ereport(ERROR,
+						(errcode(ERRCODE_DATATYPE_MISMATCH),
+						 errmsg("cannot use RETURNING type %s in JSON_SERIALIZE",
+								format_type_be(returning->typid)),
+						 errhint("Try returning a string type or bytea")));
+		}
+	}
 	else
 	{
 		/* RETURNING TEXT FORMAT JSON is by default */

src/test/regress/expected/sqljson.out

@@ -302,12 +302,22 @@ SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING bytea);
  \x7b20226122203a2031207d20
 (1 row)
 
+SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING varchar);
+ json_serialize 
+----------------
+ { "a" : 1 } 
+(1 row)
+
 SELECT pg_typeof(JSON_SERIALIZE(NULL));
  pg_typeof 
 -----------
  text
 (1 row)
 
+-- only string types or bytea allowed
+SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING jsonb);
+ERROR:  cannot use RETURNING type jsonb in JSON_SERIALIZE
+HINT:  Try returning a string type or bytea
 EXPLAIN (VERBOSE, COSTS OFF) SELECT JSON_SERIALIZE('{}');
                      QUERY PLAN                      
 -----------------------------------------------------

src/test/regress/sql/sqljson.sql

@@ -60,8 +60,13 @@ SELECT JSON_SERIALIZE('{ "a" : 1 } ');
 SELECT JSON_SERIALIZE('1');
 SELECT JSON_SERIALIZE('1' FORMAT JSON);
 SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING bytea);
+SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING varchar);
 SELECT pg_typeof(JSON_SERIALIZE(NULL));
 
+-- only string types or bytea allowed
+SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING jsonb);
+
+
 EXPLAIN (VERBOSE, COSTS OFF) SELECT JSON_SERIALIZE('{}');
 EXPLAIN (VERBOSE, COSTS OFF) SELECT JSON_SERIALIZE('{}' RETURNING bytea);