Add 'noError' argument to encoding conversion functions.

With the 'noError' argument, you can try to convert a buffer without
knowing the character boundaries beforehand. The functions now need to
return the number of input bytes successfully converted.

This is is a backwards-incompatible change, if you have created a custom
encoding conversion with CREATE CONVERSION. This adds a check to
pg_upgrade for that, refusing the upgrade if there are any user-defined
encoding conversions. Custom conversions are very rare, there are no
commonly used extensions that I know of that uses that feature. No other
objects can depend on conversions, so if you do have one, you can fairly
easily drop it before upgrading, and recreate it after the upgrade with
an updated version.

Add regression tests for built-in encoding conversions. This doesn't cover
every conversion, but it covers all the internal functions in conv.c that
are used to implement the conversions.

Reviewed-by: John Naylor
Discussion: https://www.postgresql.org/message-id/e7861509-3960-538a-9025-b75a61188e01%40iki.fi

Author: hlinnaka

doc/src/sgml/ref/create_conversion.sgml

@@ -117,9 +117,15 @@ conv_proc(
     integer,  -- destination encoding ID
     cstring,  -- source string (null terminated C string)
     internal, -- destination (fill with a null terminated C string)
-    integer   -- source string length
-) RETURNS void;
-</programlisting></para>
+    integer,  -- source string length
+    boolean   -- if true, don't throw an error if conversion fails
+) RETURNS integer;
+</programlisting>
+       The return value is the number of source bytes that were successfully
+       converted. If the last argument is false, the function must throw an
+       error on invalid input, and the return value is always equal to the
+       source string length.
+      </para>
      </listitem>
     </varlistentry>
    </variablelist>

src/backend/commands/conversioncmds.c

@@ -45,8 +45,9 @@ CreateConversionCommand(CreateConversionStmt *stmt)
 	const char *from_encoding_name = stmt->for_encoding_name;
 	const char *to_encoding_name = stmt->to_encoding_name;
 	List	   *func_name = stmt->func_name;
-	static const Oid funcargs[] = {INT4OID, INT4OID, CSTRINGOID, INTERNALOID, INT4OID};
+	static const Oid funcargs[] = {INT4OID, INT4OID, CSTRINGOID, INTERNALOID, INT4OID, BOOLOID};
 	char		result[1];
+	Datum		funcresult;
 
 	/* Convert list of names to a name and namespace */
 	namespaceId = QualifiedNameGetCreationNamespace(stmt->conversion_name,
@@ -92,12 +93,12 @@ CreateConversionCommand(CreateConversionStmt *stmt)
 	funcoid = LookupFuncName(func_name, sizeof(funcargs) / sizeof(Oid),
 							 funcargs, false);
 
-	/* Check it returns VOID, else it's probably the wrong function */
-	if (get_func_rettype(funcoid) != VOIDOID)
+	/* Check it returns int4, else it's probably the wrong function */
+	if (get_func_rettype(funcoid) != INT4OID)
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
 				 errmsg("encoding conversion function %s must return type %s",
-						NameListToString(func_name), "void")));
+						NameListToString(func_name), "integer")));
 
 	/* Check we have EXECUTE rights for the function */
 	aclresult = pg_proc_aclcheck(funcoid, GetUserId(), ACL_EXECUTE);
@@ -111,12 +112,23 @@ CreateConversionCommand(CreateConversionStmt *stmt)
 	 * string; the conversion function should throw an error if it can't
 	 * perform the requested conversion.
 	 */
-	OidFunctionCall5(funcoid,
-					 Int32GetDatum(from_encoding),
-					 Int32GetDatum(to_encoding),
-					 CStringGetDatum(""),
-					 CStringGetDatum(result),
-					 Int32GetDatum(0));
+	funcresult = OidFunctionCall6(funcoid,
+								  Int32GetDatum(from_encoding),
+								  Int32GetDatum(to_encoding),
+								  CStringGetDatum(""),
+								  CStringGetDatum(result),
+								  Int32GetDatum(0),
+								  BoolGetDatum(false));
+
+	/*
+	 * The function should return 0 for empty input. Might as well check that,
+	 * too.
+	 */
+	if (DatumGetInt32(funcresult) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+				 errmsg("encoding conversion function %s returned incorrect result for empty input",
+						NameListToString(func_name))));
 
 	/*
 	 * All seem ok, go ahead (possible failure would be a duplicate conversion

src/backend/utils/error/elog.c

@@ -2271,6 +2271,8 @@ write_console(const char *line, int len)
 	 * Conversion on non-win32 platforms is not implemented yet. It requires
 	 * non-throw version of pg_do_encoding_conversion(), that converts
 	 * unconvertable characters to '?' without errors.
+	 *
+	 * XXX: We have a no-throw version now. It doesn't convert to '?' though.
 	 */
 #endif