Fix possible buffer overrun in contrib/pg_trgm.

Allow for the possibility that folding a string to lower case makes it
longer (due to replacing a character with a longer multibyte character).
This doesn't change the number of trigrams that will be extracted, but
it does affect the required size of an intermediate buffer in
generate_trgm().  Per bug #8821 from Ufuk Kayserilioglu.

Also install some checks that the input string length is not so large
as to cause overflow in the calculations of palloc request sizes.

Back-patch to all supported versions.

Author: tglsfdc

contrib/pg_trgm/trgm_op.c

@@ -9,6 +9,7 @@
 
 #include "catalog/pg_type.h"
 #include "tsearch/ts_locale.h"
+#include "utils/memutils.h"
 
 
 PG_MODULE_MAGIC;
@@ -188,6 +189,18 @@ generate_trgm(char *str, int slen)
 	char	   *bword,
 			   *eword;
 
+	/*
+	 * Guard against possible overflow in the palloc requests below.  (We
+	 * don't worry about the additive constants, since palloc can detect
+	 * requests that are a little above MaxAllocSize --- we just need to
+	 * prevent integer overflow in the multiplications.)
+	 */
+	if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
+		(Size) slen >= (MaxAllocSize / pg_database_encoding_max_length()))
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+				 errmsg("out of memory")));
+
 	trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) *3);
 	trg->flag = ARRKEY;
 	SET_VARSIZE(trg, TRGMHDRSIZE);
@@ -197,7 +210,8 @@ generate_trgm(char *str, int slen)
 
 	tptr = GETARR(trg);
 
-	buf = palloc(sizeof(char) * (slen + 4));
+	/* Allocate a buffer for case-folded, blank-padded words */
+	buf = (char *) palloc(slen * pg_database_encoding_max_length() + 4);
 
 	if (LPADDING > 0)
 	{
@@ -221,6 +235,7 @@ generate_trgm(char *str, int slen)
 #ifdef IGNORECASE
 		pfree(bword);
 #endif
+
 		buf[LPADDING + bytelen] = ' ';
 		buf[LPADDING + bytelen + 1] = ' ';
 
@@ -236,7 +251,10 @@ generate_trgm(char *str, int slen)
 	if ((len = tptr - GETARR(trg)) == 0)
 		return trg;
 
-	if (len > 0)
+	/*
+	 * Make trigrams unique.
+	 */
+	if (len > 1)
 	{
 		qsort((void *) GETARR(trg), len, sizeof(trgm), comp_trgm);
 		len = unique_array(GETARR(trg), len);
@@ -419,6 +437,18 @@ generate_wildcard_trgm(const char *str, int slen)
 				bytelen;
 	const char *eword;
 
+	/*
+	 * Guard against possible overflow in the palloc requests below.  (We
+	 * don't worry about the additive constants, since palloc can detect
+	 * requests that are a little above MaxAllocSize --- we just need to
+	 * prevent integer overflow in the multiplications.)
+	 */
+	if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
+		(Size) slen >= (MaxAllocSize / pg_database_encoding_max_length()))
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+				 errmsg("out of memory")));
+
 	trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) *3);
 	trg->flag = ARRKEY;
 	SET_VARSIZE(trg, TRGMHDRSIZE);
@@ -428,6 +458,7 @@ generate_wildcard_trgm(const char *str, int slen)
 
 	tptr = GETARR(trg);
 
+	/* Allocate a buffer for blank-padded, but not yet case-folded, words */
 	buf = palloc(sizeof(char) * (slen + 4));
 
 	/*
@@ -448,6 +479,7 @@ generate_wildcard_trgm(const char *str, int slen)
 		 * count trigrams
 		 */
 		tptr = make_trigrams(tptr, buf2, bytelen, charlen);
+
 #ifdef IGNORECASE
 		pfree(buf2);
 #endif
@@ -461,7 +493,7 @@ generate_wildcard_trgm(const char *str, int slen)
 	/*
 	 * Make trigrams unique.
 	 */
-	if (len > 0)
+	if (len > 1)
 	{
 		qsort((void *) GETARR(trg), len, sizeof(trgm), comp_trgm);
 		len = unique_array(GETARR(trg), len);