Support "x IS NOT NULL" clauses as indexscan conditions. This turns out

to be just a minor extension of the previous patch that made "x IS NULL"
indexable, because we can treat the IS NOT NULL condition as if it were
"x < NULL" or "x > NULL" (depending on the index's NULLS FIRST/LAST option),
just like IS NULL is treated like "x = NULL".  Aside from any possible
usefulness in its own right, this is an important improvement for
index-optimized MAX/MIN aggregates: it is now reliably possible to get
a column's min or max value cheaply, even when there are a lot of nulls
cluttering the interesting end of the index.

Author: tglsfdc

doc/src/sgml/catalogs.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/catalogs.sgml,v 2.215 2009/12/29 20:11:42 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/catalogs.sgml,v 2.216 2010/01/01 21:53:48 tgl Exp $ -->
 <!--
  Documentation of the system catalogs, directed toward PostgreSQL developers
  -->
@@ -466,7 +466,7 @@
       <entry><structfield>amsearchnulls</structfield></entry>
       <entry><type>bool</type></entry>
       <entry></entry>
-      <entry>Does the access method support IS NULL searches?</entry>
+      <entry>Does the access method support IS NULL/NOT NULL searches?</entry>
      </row>
 
      <row>

doc/src/sgml/indexam.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/indexam.sgml,v 2.31 2009/07/29 20:56:17 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/indexam.sgml,v 2.32 2010/01/01 21:53:49 tgl Exp $ -->
 
 <chapter id="indexam">
  <title>Index Access Method Interface Definition</title>
@@ -134,7 +134,8 @@
    null values.  An index access method that sets
    <structfield>amindexnulls</structfield> may also set
    <structfield>amsearchnulls</structfield>, indicating that it supports
-   <literal>IS NULL</> clauses as search conditions.
+   <literal>IS NULL</> and <literal>IS NOT NULL</> clauses as search
+   conditions.
   </para>
 
  </sect1>

doc/src/sgml/indices.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/indices.sgml,v 1.79 2009/08/07 20:54:31 alvherre Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/indices.sgml,v 1.80 2010/01/01 21:53:49 tgl Exp $ -->
 
 <chapter id="indexes">
  <title id="indexes-title">Indexes</title>
@@ -147,8 +147,8 @@ CREATE INDEX test1_id_index ON test1 (id);
 
    Constructs equivalent to combinations of these operators, such as
    <literal>BETWEEN</> and <literal>IN</>, can also be implemented with
-   a B-tree index search.  Also, an <literal>IS NULL</> condition on
-   an index column can be used with a B-tree index.
+   a B-tree index search.  Also, an <literal>IS NULL</> or <literal>IS NOT
+   NULL</> condition on an index column can be used with a B-tree index.
   </para>
 
   <para>
@@ -180,8 +180,7 @@ CREATE INDEX test1_id_index ON test1 (id);
    Hash indexes can only handle simple equality comparisons.
    The query planner will consider using a hash index whenever an
    indexed column is involved in a comparison using the
-   <literal>=</literal> operator.  (Hash indexes do not support
-   <literal>IS NULL</> searches.)
+   <literal>=</literal> operator.
    The following command is used to create a hash index:
 <synopsis>
 CREATE INDEX <replaceable>name</replaceable> ON <replaceable>table</replaceable> USING hash (<replaceable>column</replaceable>);
@@ -1025,7 +1024,7 @@ SELECT am.amname AS index_method,
      real statistics, some default values are assumed, which are
      almost certain to be inaccurate.  Examining an application's
      index usage without having run <command>ANALYZE</command> is
-     therefore a lost cause. 
+     therefore a lost cause.
      See <xref linkend="vacuum-for-statistics" endterm="vacuum-for-statistics-title">
      and <xref linkend="autovacuum" endterm="autovacuum-title"> for more information.
     </para>

src/backend/access/common/scankey.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/common/scankey.c,v 1.32 2009/01/01 17:23:34 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/common/scankey.c,v 1.33 2010/01/01 21:53:49 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -21,7 +21,7 @@
  * ScanKeyEntryInitialize
  *		Initializes a scan key entry given all the field values.
  *		The target procedure is specified by OID (but can be invalid
- *		if SK_SEARCHNULL is set).
+ *		if SK_SEARCHNULL or SK_SEARCHNOTNULL is set).
  *
  * Note: CurrentMemoryContext at call should be as long-lived as the ScanKey
  * itself, because that's what will be used for any subsidiary info attached
@@ -45,7 +45,7 @@ ScanKeyEntryInitialize(ScanKey entry,
 		fmgr_info(procedure, &entry->sk_func);
 	else
 	{
-		Assert(flags & SK_SEARCHNULL);
+		Assert(flags & (SK_SEARCHNULL | SK_SEARCHNOTNULL));
 		MemSet(&entry->sk_func, 0, sizeof(entry->sk_func));
 	}
 }

src/backend/access/gist/gistget.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/gist/gistget.c,v 1.82 2009/10/08 22:34:57 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/gist/gistget.c,v 1.83 2010/01/01 21:53:49 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -413,14 +413,20 @@ gistindex_keytest(IndexTuple tuple,
 		{
 			/*
 			 * On non-leaf page we can't conclude that child hasn't NULL
-			 * values because of assumption in GiST: uinon (VAL, NULL) is VAL
-			 * But if on non-leaf page key IS  NULL then all childs has NULL.
+			 * values because of assumption in GiST: union (VAL, NULL) is VAL.
+			 * But if on non-leaf page key IS NULL, then all children are NULL.
 			 */
-
-			Assert(key->sk_flags & SK_SEARCHNULL);
-
-			if (GistPageIsLeaf(p) && !isNull)
-				return false;
+			if (key->sk_flags & SK_SEARCHNULL)
+			{
+				if (GistPageIsLeaf(p) && !isNull)
+					return false;
+			}
+			else
+			{
+				Assert(key->sk_flags & SK_SEARCHNOTNULL);
+				if (isNull)
+					return false;
+			}
 		}
 		else if (isNull)
 		{

src/backend/access/gist/gistscan.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/gist/gistscan.c,v 1.76 2009/06/11 14:48:53 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/gist/gistscan.c,v 1.77 2010/01/01 21:53:49 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -92,15 +92,18 @@ gistrescan(PG_FUNCTION_ARGS)
 		 * field.
 		 *
 		 * Next, if any of keys is a NULL and that key is not marked with
-		 * SK_SEARCHNULL then nothing can be found.
+		 * SK_SEARCHNULL/SK_SEARCHNOTNULL then nothing can be found (ie,
+		 * we assume all indexable operators are strict).
 		 */
 		for (i = 0; i < scan->numberOfKeys; i++)
 		{
-			scan->keyData[i].sk_func = so->giststate->consistentFn[scan->keyData[i].sk_attno - 1];
+			ScanKey		skey = &(scan->keyData[i]);
 
-			if (scan->keyData[i].sk_flags & SK_ISNULL)
+			skey->sk_func = so->giststate->consistentFn[skey->sk_attno - 1];
+
+			if (skey->sk_flags & SK_ISNULL)
 			{
-				if ((scan->keyData[i].sk_flags & SK_SEARCHNULL) == 0)
+				if (!(skey->sk_flags & (SK_SEARCHNULL | SK_SEARCHNOTNULL)))
 					so->qual_ok = false;
 			}
 		}

src/backend/access/nbtree/nbtutils.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtutils.c,v 1.94 2009/10/08 22:34:57 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtutils.c,v 1.95 2010/01/01 21:53:49 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -276,6 +276,11 @@ _bt_preprocess_keys(IndexScanDesc scan)
 		 * in any particular strategy in this case, so set it to
 		 * BTEqualStrategyNumber --- we can treat IS NULL as an equality
 		 * operator for purposes of search strategy.
+		 *
+		 * Likewise, "x IS NOT NULL" is supported.  We treat that as either
+		 * "less than NULL" in a NULLS LAST index, or "greater than NULL"
+		 * in a NULLS FIRST index.  However, we have to flip those around in
+		 * a DESC index, to allow for the re-flipping that occurs elsewhere.
 		 */
 		if (cur->sk_flags & SK_ISNULL)
 		{
@@ -284,6 +289,21 @@ _bt_preprocess_keys(IndexScanDesc scan)
 				cur->sk_strategy = BTEqualStrategyNumber;
 				cur->sk_subtype = InvalidOid;
 			}
+			else if (cur->sk_flags & SK_SEARCHNOTNULL)
+			{
+				switch (indoption[cur->sk_attno - 1] &
+						(INDOPTION_DESC | INDOPTION_NULLS_FIRST))
+				{
+					case 0:		/* ASC / NULLS LAST */
+					case INDOPTION_DESC | INDOPTION_NULLS_FIRST:
+						cur->sk_strategy = BTLessStrategyNumber;
+						break;
+					default:
+						cur->sk_strategy = BTGreaterStrategyNumber;
+						break;
+				}
+				cur->sk_subtype = InvalidOid;
+			}
 			else
 				so->qual_ok = false;
 		}
@@ -320,7 +340,7 @@ _bt_preprocess_keys(IndexScanDesc scan)
 	{
 		if (i < numberOfKeys)
 		{
-			/* See comments above about NULLs and IS NULL handling. */
+			/* See comments above about NULLs and IS NULL/NOT NULL handling */
 			/* Note: we assume SK_ISNULL is never set in a row header key */
 			if (cur->sk_flags & SK_ISNULL)
 			{
@@ -329,6 +349,21 @@ _bt_preprocess_keys(IndexScanDesc scan)
 					cur->sk_strategy = BTEqualStrategyNumber;
 					cur->sk_subtype = InvalidOid;
 				}
+				else if (cur->sk_flags & SK_SEARCHNOTNULL)
+				{
+					switch (indoption[cur->sk_attno - 1] &
+							(INDOPTION_DESC | INDOPTION_NULLS_FIRST))
+					{
+						case 0:		/* ASC / NULLS LAST */
+						case INDOPTION_DESC | INDOPTION_NULLS_FIRST:
+							cur->sk_strategy = BTLessStrategyNumber;
+							break;
+						default:
+							cur->sk_strategy = BTGreaterStrategyNumber;
+							break;
+					}
+					cur->sk_subtype = InvalidOid;
+				}
 				else
 				{
 					so->qual_ok = false;
@@ -365,13 +400,6 @@ _bt_preprocess_keys(IndexScanDesc scan)
 					if (!chk || j == (BTEqualStrategyNumber - 1))
 						continue;
 
-					/* IS NULL together with any other predicate must fail */
-					if (eq->sk_flags & SK_SEARCHNULL)
-					{
-						so->qual_ok = false;
-						return;
-					}
-
 					if (_bt_compare_scankey_args(scan, chk, eq, chk,
 												 &test_result))
 					{
@@ -484,23 +512,6 @@ _bt_preprocess_keys(IndexScanDesc scan)
 		else
 		{
 			/* yup, keep only the more restrictive key */
-
-			/* if either arg is NULL, don't try to compare */
-			if ((cur->sk_flags | xform[j]->sk_flags) & SK_ISNULL)
-			{
-				/* at least one of them must be an IS NULL clause */
-				Assert(j == (BTEqualStrategyNumber - 1));
-				Assert((cur->sk_flags | xform[j]->sk_flags) & SK_SEARCHNULL);
-				/* if one is and one isn't, the search must fail */
-				if ((cur->sk_flags ^ xform[j]->sk_flags) & SK_SEARCHNULL)
-				{
-					so->qual_ok = false;
-					return;
-				}
-				/* we have duplicate IS NULL clauses, ignore the newer one */
-				continue;
-			}
-
 			if (_bt_compare_scankey_args(scan, cur, cur, xform[j],
 										 &test_result))
 			{
@@ -534,8 +545,7 @@ _bt_preprocess_keys(IndexScanDesc scan)
 }
 
 /*
- * Compare two scankey values using a specified operator.  Both values
- * must be already known non-NULL.
+ * Compare two scankey values using a specified operator.
  *
  * The test we want to perform is logically "leftarg op rightarg", where
  * leftarg and rightarg are the sk_argument values in those ScanKeys, and
@@ -555,8 +565,7 @@ _bt_preprocess_keys(IndexScanDesc scan)
  *
  * Note: this routine needs to be insensitive to any DESC option applied
  * to the index column.  For example, "x < 4" is a tighter constraint than
- * "x < 5" regardless of which way the index is sorted.  We don't worry about
- * NULLS FIRST/LAST either, since the given values are never nulls.
+ * "x < 5" regardless of which way the index is sorted.
  */
 static bool
 _bt_compare_scankey_args(IndexScanDesc scan, ScanKey op,
@@ -571,6 +580,64 @@ _bt_compare_scankey_args(IndexScanDesc scan, ScanKey op,
 				cmp_op;
 	StrategyNumber strat;
 
+	/*
+	 * First, deal with cases where one or both args are NULL.  This should
+	 * only happen when the scankeys represent IS NULL/NOT NULL conditions.
+	 */
+	if ((leftarg->sk_flags | rightarg->sk_flags) & SK_ISNULL)
+	{
+		bool		leftnull,
+					rightnull;
+
+		if (leftarg->sk_flags & SK_ISNULL)
+		{
+			Assert(leftarg->sk_flags & (SK_SEARCHNULL | SK_SEARCHNOTNULL));
+			leftnull = true;
+		}
+		else
+			leftnull = false;
+		if (rightarg->sk_flags & SK_ISNULL)
+		{
+			Assert(rightarg->sk_flags & (SK_SEARCHNULL | SK_SEARCHNOTNULL));
+			rightnull = true;
+		}
+		else
+			rightnull = false;
+
+		/*
+		 * We treat NULL as either greater than or less than all other values.
+		 * Since true > false, the tests below work correctly for NULLS LAST
+		 * logic.  If the index is NULLS FIRST, we need to flip the strategy.
+		 */
+		strat = op->sk_strategy;
+		if (op->sk_flags & SK_BT_NULLS_FIRST)
+			strat = BTCommuteStrategyNumber(strat);
+
+		switch (strat)
+		{
+			case BTLessStrategyNumber:
+				*result = (leftnull < rightnull);
+				break;
+			case BTLessEqualStrategyNumber:
+				*result = (leftnull <= rightnull);
+				break;
+			case BTEqualStrategyNumber:
+				*result = (leftnull == rightnull);
+				break;
+			case BTGreaterEqualStrategyNumber:
+				*result = (leftnull >= rightnull);
+				break;
+			case BTGreaterStrategyNumber:
+				*result = (leftnull > rightnull);
+				break;
+			default:
+				elog(ERROR, "unrecognized StrategyNumber: %d", (int) strat);
+				*result = false;		/* keep compiler quiet */
+				break;
+		}
+		return true;
+	}
+
 	/*
 	 * The opfamily we need to worry about is identified by the index column.
 	 */
@@ -844,11 +911,18 @@ _bt_checkkeys(IndexScanDesc scan,
 
 		if (key->sk_flags & SK_ISNULL)
 		{
-			/* Handle IS NULL tests */
-			Assert(key->sk_flags & SK_SEARCHNULL);
-
-			if (isNull)
-				continue;		/* tuple satisfies this qual */
+			/* Handle IS NULL/NOT NULL tests */
+			if (key->sk_flags & SK_SEARCHNULL)
+			{
+				if (isNull)
+					continue;		/* tuple satisfies this qual */
+			}
+			else
+			{
+				Assert(key->sk_flags & SK_SEARCHNOTNULL);
+				if (!isNull)
+					continue;		/* tuple satisfies this qual */
+			}
 
 			/*
 			 * Tuple fails this qual.  If it's a required qual for the current