Support LIKE and ILIKE index searches via contrib/pg_trgm indexes.

Unlike Btree-based LIKE optimization, this works for non-left-anchored
search patterns.  The effectiveness of the search depends on how many
trigrams can be extracted from the pattern.  (The worst case, with no
trigrams, degrades to a full-table scan, so this isn't a panacea.  But
it can be very useful.)

Alexander Korotkov, reviewed by Jan Urbanski

Author: tglsfdc

contrib/pg_trgm/expected/pg_trgm.out

@@ -3467,3 +3467,93 @@ select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu198
  qwertyu0988 | 0.333333
 (1 row)
 
+create table test2(t text);
+insert into test2 values ('abcdef');
+insert into test2 values ('quark');
+create index test2_idx_gin on test2 using gin (t gin_trgm_ops);
+set enable_seqscan=off;
+explain (costs off)
+  select * from test2 where t like '%BCD%';
+                QUERY PLAN                
+------------------------------------------
+ Bitmap Heap Scan on test2
+   Recheck Cond: (t ~~ '%BCD%'::text)
+   ->  Bitmap Index Scan on test2_idx_gin
+         Index Cond: (t ~~ '%BCD%'::text)
+(4 rows)
+
+explain (costs off)
+  select * from test2 where t ilike '%BCD%';
+                QUERY PLAN                 
+-------------------------------------------
+ Bitmap Heap Scan on test2
+   Recheck Cond: (t ~~* '%BCD%'::text)
+   ->  Bitmap Index Scan on test2_idx_gin
+         Index Cond: (t ~~* '%BCD%'::text)
+(4 rows)
+
+select * from test2 where t like '%BCD%';
+ t 
+---
+(0 rows)
+
+select * from test2 where t like '%bcd%';
+   t    
+--------
+ abcdef
+(1 row)
+
+select * from test2 where t ilike '%BCD%';
+   t    
+--------
+ abcdef
+(1 row)
+
+select * from test2 where t ilike 'qua%';
+   t   
+-------
+ quark
+(1 row)
+
+drop index test2_idx_gin;
+create index test2_idx_gist on test2 using gist (t gist_trgm_ops);
+set enable_seqscan=off;
+explain (costs off)
+  select * from test2 where t like '%BCD%';
+                QUERY PLAN                
+------------------------------------------
+ Index Scan using test2_idx_gist on test2
+   Index Cond: (t ~~ '%BCD%'::text)
+(2 rows)
+
+explain (costs off)
+  select * from test2 where t ilike '%BCD%';
+                QUERY PLAN                
+------------------------------------------
+ Index Scan using test2_idx_gist on test2
+   Index Cond: (t ~~* '%BCD%'::text)
+(2 rows)
+
+select * from test2 where t like '%BCD%';
+ t 
+---
+(0 rows)
+
+select * from test2 where t like '%bcd%';
+   t    
+--------
+ abcdef
+(1 row)
+
+select * from test2 where t ilike '%BCD%';
+   t    
+--------
+ abcdef
+(1 row)
+
+select * from test2 where t ilike 'qua%';
+   t   
+-------
+ quark
+(1 row)
+

contrib/pg_trgm/pg_trgm.sql.in

@@ -113,6 +113,8 @@ FOR TYPE text USING gist
 AS
         OPERATOR        1       % (text, text),
         OPERATOR        2       <-> (text, text) FOR ORDER BY pg_catalog.float_ops,
+        OPERATOR        3       pg_catalog.~~ (text, text),
+        OPERATOR        4       pg_catalog.~~* (text, text),
         FUNCTION        1       gtrgm_consistent (internal, text, int, oid, internal),
         FUNCTION        2       gtrgm_union (bytea, internal),
         FUNCTION        3       gtrgm_compress (internal),
@@ -124,17 +126,17 @@ AS
         STORAGE         gtrgm;
 
 -- support functions for gin
-CREATE OR REPLACE FUNCTION gin_extract_trgm(text, internal)
+CREATE OR REPLACE FUNCTION gin_extract_value_trgm(text, internal)
 RETURNS internal
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
-CREATE OR REPLACE FUNCTION gin_extract_trgm(text, internal, int2, internal, internal)
+CREATE OR REPLACE FUNCTION gin_extract_query_trgm(text, internal, int2, internal, internal, internal, internal)
 RETURNS internal
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
-CREATE OR REPLACE FUNCTION gin_trgm_consistent(internal, int2, text, int4, internal, internal)
+CREATE OR REPLACE FUNCTION gin_trgm_consistent(internal, int2, text, int4, internal, internal, internal, internal)
 RETURNS bool
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
@@ -144,8 +146,10 @@ CREATE OPERATOR CLASS gin_trgm_ops
 FOR TYPE text USING gin
 AS
         OPERATOR        1       % (text, text),
+        OPERATOR        3       pg_catalog.~~ (text, text),
+        OPERATOR        4       pg_catalog.~~* (text, text),
         FUNCTION        1       btint4cmp (int4, int4),
-        FUNCTION        2       gin_extract_trgm (text, internal),
-        FUNCTION        3       gin_extract_trgm (text, internal, int2, internal, internal),
-        FUNCTION        4       gin_trgm_consistent (internal, int2, text, int4, internal, internal),
+        FUNCTION        2       gin_extract_value_trgm (text, internal),
+        FUNCTION        3       gin_extract_query_trgm (text, internal, int2, internal, internal, internal, internal),
+        FUNCTION        4       gin_trgm_consistent (internal, int2, text, int4, internal, internal, internal, internal),
         STORAGE         int4;

contrib/pg_trgm/sql/pg_trgm.sql

@@ -45,3 +45,28 @@ set enable_seqscan=off;
 select t,similarity(t,'qwertyu0988') as sml from test_trgm where t % 'qwertyu0988' order by sml desc, t;
 select t,similarity(t,'gwertyu0988') as sml from test_trgm where t % 'gwertyu0988' order by sml desc, t;
 select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu1988' order by sml desc, t;
+
+create table test2(t text);
+insert into test2 values ('abcdef');
+insert into test2 values ('quark');
+create index test2_idx_gin on test2 using gin (t gin_trgm_ops);
+set enable_seqscan=off;
+explain (costs off)
+  select * from test2 where t like '%BCD%';
+explain (costs off)
+  select * from test2 where t ilike '%BCD%';
+select * from test2 where t like '%BCD%';
+select * from test2 where t like '%bcd%';
+select * from test2 where t ilike '%BCD%';
+select * from test2 where t ilike 'qua%';
+drop index test2_idx_gin;
+create index test2_idx_gist on test2 using gist (t gist_trgm_ops);
+set enable_seqscan=off;
+explain (costs off)
+  select * from test2 where t like '%BCD%';
+explain (costs off)
+  select * from test2 where t ilike '%BCD%';
+select * from test2 where t like '%BCD%';
+select * from test2 where t like '%bcd%';
+select * from test2 where t ilike '%BCD%';
+select * from test2 where t ilike 'qua%';

contrib/pg_trgm/trgm.h

@@ -13,12 +13,21 @@
 #define LPADDING		2
 #define RPADDING		1
 #define KEEPONLYALNUM
+/*
+ * Caution: IGNORECASE macro means that trigrams are case-insensitive.
+ * If this macro is disabled, the ~~* operator must be removed from the
+ * operator classes, because we can't handle case-insensitive wildcard search
+ * with case-sensitive trigrams.  Failure to do this will result in "cannot
+ * handle ~~* with case-sensitive trigrams" errors.
+ */
 #define IGNORECASE
 #define DIVUNION
 
 /* operator strategy numbers */
-#define	SimilarityStrategyNumber	1
-#define	DistanceStrategyNumber		2
+#define SimilarityStrategyNumber	1
+#define DistanceStrategyNumber		2
+#define LikeStrategyNumber			3
+#define ILikeStrategyNumber			4
 
 
 typedef char trgm[3];
@@ -40,7 +49,10 @@ uint32		trgm2int(trgm *ptr);
 #else
 #define ISPRINTABLECHAR(a)	( isascii( *(unsigned char*)(a) ) && isprint( *(unsigned char*)(a) ) )
 #endif
-#define ISPRINTABLETRGM(t)	( ISPRINTABLECHAR( ((char*)t) ) && ISPRINTABLECHAR( ((char*)t)+1 ) && ISPRINTABLECHAR( ((char*)t)+2 ) )
+#define ISPRINTABLETRGM(t)	( ISPRINTABLECHAR( ((char*)(t)) ) && ISPRINTABLECHAR( ((char*)(t))+1 ) && ISPRINTABLECHAR( ((char*)(t))+2 ) )
+
+#define ISESCAPECHAR(x) (*(x) == '\\') /* Wildcard escape character */
+#define ISWILDCARDCHAR(x) (*(x) == '_' || *(x) == '%')  /* Wildcard meta-character */
 
 typedef struct
 {
@@ -65,7 +77,7 @@ typedef char *BITVECP;
 			for(i=0;i<SIGLEN;i++)
 
 #define GETBYTE(x,i) ( *( (BITVECP)(x) + (int)( (i) / BITBYTE ) ) )
-#define GETBITBYTE(x,i) ( ((char)(x)) >> i & 0x01 )
+#define GETBITBYTE(x,i) ( (((char)(x)) >> (i)) & 0x01 )
 #define CLRBIT(x,i)   GETBYTE(x,i) &= ~( 0x01 << ( (i) % BITBYTE ) )
 #define SETBIT(x,i)   GETBYTE(x,i) |=  ( 0x01 << ( (i) % BITBYTE ) )
 #define GETBIT(x,i) ( (GETBYTE(x,i) >> ( (i) % BITBYTE )) & 0x01 )
@@ -89,6 +101,8 @@ typedef char *BITVECP;
 extern float4 trgm_limit;
 
 TRGM	   *generate_trgm(char *str, int slen);
+TRGM	   *generate_wildcard_trgm(const char *str, int slen);
 float4		cnt_sml(TRGM *trg1, TRGM *trg2);
+bool		trgm_contained_by(TRGM *trg1, TRGM *trg2);
 
 #endif /* __TRGM_H__ */

contrib/pg_trgm/trgm_gin.c

@@ -7,6 +7,7 @@
 
 #include "access/gin.h"
 #include "access/itup.h"
+#include "access/skey.h"
 #include "access/tuptoaster.h"
 #include "storage/bufpage.h"
 #include "utils/array.h"
@@ -16,14 +17,31 @@
 PG_FUNCTION_INFO_V1(gin_extract_trgm);
 Datum		gin_extract_trgm(PG_FUNCTION_ARGS);
 
+PG_FUNCTION_INFO_V1(gin_extract_value_trgm);
+Datum		gin_extract_value_trgm(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(gin_extract_query_trgm);
+Datum		gin_extract_query_trgm(PG_FUNCTION_ARGS);
+
 PG_FUNCTION_INFO_V1(gin_trgm_consistent);
 Datum		gin_trgm_consistent(PG_FUNCTION_ARGS);
 
 /*
- * This function is used as both extractValue and extractQuery
+ * This function can only be called if a pre-9.1 version of the GIN operator
+ * class definition is present in the catalogs (probably as a consequence
+ * of upgrade-in-place).  Complain.
  */
 Datum
 gin_extract_trgm(PG_FUNCTION_ARGS)
+{
+	ereport(ERROR,
+			(errmsg("GIN operator class for pg_trgm is out of date"),
+			 errhint("Please drop and re-create the pg_trgm catalog entries.")));
+	PG_RETURN_NULL();
+}
+
+Datum
+gin_extract_value_trgm(PG_FUNCTION_ARGS)
 {
 	text	   *val = (text *) PG_GETARG_TEXT_P(0);
 	int32	   *nentries = (int32 *) PG_GETARG_POINTER(1);
@@ -57,34 +75,124 @@ gin_extract_trgm(PG_FUNCTION_ARGS)
 	PG_RETURN_POINTER(entries);
 }
 
+Datum
+gin_extract_query_trgm(PG_FUNCTION_ARGS)
+{
+	text	   *val = (text *) PG_GETARG_TEXT_P(0);
+	int32	   *nentries = (int32 *) PG_GETARG_POINTER(1);
+	StrategyNumber strategy = PG_GETARG_UINT16(2);
+	/* bool	  **pmatch = (bool **) PG_GETARG_POINTER(3); */
+	/* Pointer    *extra_data = (Pointer *) PG_GETARG_POINTER(4); */
+	/* bool	  **nullFlags = (bool **) PG_GETARG_POINTER(5); */
+	int32      *searchMode = (int32 *) PG_GETARG_POINTER(6);
+	Datum	   *entries = NULL;
+	TRGM	   *trg;
+	int32		trglen;
+	trgm	   *ptr;
+	int32		i;
+
+	switch (strategy)
+	{
+		case SimilarityStrategyNumber:
+			trg = generate_trgm(VARDATA(val), VARSIZE(val) - VARHDRSZ);
+			break;
+		case ILikeStrategyNumber:
+#ifndef IGNORECASE
+			elog(ERROR, "cannot handle ~~* with case-sensitive trigrams");
+#endif
+			/* FALL THRU */
+		case LikeStrategyNumber:
+			/*
+			 * For wildcard search we extract all the trigrams that every
+			 * potentially-matching string must include.
+			 */
+			trg = generate_wildcard_trgm(VARDATA(val), VARSIZE(val) - VARHDRSZ);
+			break;
+		default:
+			elog(ERROR, "unrecognized strategy number: %d", strategy);
+			trg = NULL;		/* keep compiler quiet */
+			break;
+	}
+
+	trglen = ARRNELEM(trg);
+	*nentries = trglen;
+
+	if (trglen > 0)
+	{
+		entries = (Datum *) palloc(sizeof(Datum) * trglen);
+		ptr = GETARR(trg);
+		for (i = 0; i < trglen; i++)
+		{
+			int32	item = trgm2int(ptr);
+
+			entries[i] = Int32GetDatum(item);
+			ptr++;
+		}
+	}
+
+	/*
+	 * If no trigram was extracted then we have to scan all the index.
+	 */
+	if (trglen == 0)
+		*searchMode = GIN_SEARCH_MODE_ALL;
+
+	PG_RETURN_POINTER(entries);
+}
+
 Datum
 gin_trgm_consistent(PG_FUNCTION_ARGS)
 {
 	bool	   *check = (bool *) PG_GETARG_POINTER(0);
-	/* StrategyNumber strategy = PG_GETARG_UINT16(1); */
+	StrategyNumber strategy = PG_GETARG_UINT16(1);
 	/* text    *query = PG_GETARG_TEXT_P(2); */
 	int32		nkeys = PG_GETARG_INT32(3);
 	/* Pointer    *extra_data = (Pointer *) PG_GETARG_POINTER(4); */
 	bool	   *recheck = (bool *) PG_GETARG_POINTER(5);
-	bool		res = FALSE;
+	bool		res;
 	int32		i,
-				ntrue = 0;
+				ntrue;
 
 	/* All cases served by this function are inexact */
 	*recheck = true;
 
-	/* Count the matches */
-	for (i = 0; i < nkeys; i++)
+	switch (strategy)
 	{
-		if (check[i])
-			ntrue++;
-	}
-
+		case SimilarityStrategyNumber:
+			/* Count the matches */
+			ntrue = 0;
+			for (i = 0; i < nkeys; i++)
+			{
+				if (check[i])
+					ntrue++;
+			}
 #ifdef DIVUNION
-	res = (nkeys == ntrue) ? true : ((((((float4) ntrue) / ((float4) (nkeys - ntrue)))) >= trgm_limit) ? true : false);
+			res = (nkeys == ntrue) ? true : ((((((float4) ntrue) / ((float4) (nkeys - ntrue)))) >= trgm_limit) ? true : false);
 #else
-	res = (nkeys == 0) ? false : ((((((float4) ntrue) / ((float4) nkeys))) >= trgm_limit) ? true : false);
+			res = (nkeys == 0) ? false : ((((((float4) ntrue) / ((float4) nkeys))) >= trgm_limit) ? true : false);
 #endif
+			break;
+		case ILikeStrategyNumber:
+#ifndef IGNORECASE
+			elog(ERROR, "cannot handle ~~* with case-sensitive trigrams");
+#endif
+			/* FALL THRU */
+		case LikeStrategyNumber:
+			/* Check if all extracted trigrams are presented. */
+			res = true;
+			for (i = 0; i < nkeys; i++)
+			{
+				if (!check[i])
+				{
+					res = false;
+					break;
+				}
+			}
+			break;
+		default:
+			elog(ERROR, "unrecognized strategy number: %d", strategy);
+			res = false;		/* keep compiler quiet */
+			break;
+	}
 
 	PG_RETURN_BOOL(res);
 }