Better IDF calculation.

Author: Alexander Korotkov

src/rum_ts_utils.c

@@ -96,6 +96,7 @@ typedef struct
 		} key;
 	} data;
 	uint8		wclass;
+	float4		idf;
 	int32		pos;
 } DocRepresentation;
 
@@ -106,20 +107,13 @@ typedef struct
 }
 QueryRepresentationOperand;
 
-typedef struct
-{
-	float4			idf;
-	bool			idfloaded;
-} QueryRepresentationIDF;
-
 typedef struct
 {
 	TSQuery		query;
 	/* Used in rum_tsquery_distance() */
 	int		   *map_item_operand;
 
 	QueryRepresentationOperand	   *operandData;
-	QueryRepresentationIDF		   *operandIdf;
 	int			length;
 } QueryRepresentation;
 
@@ -1098,7 +1092,7 @@ find_wordentry(TSVector t, TSQuery q, QueryOperand *item, int32 *nitem)
 }
 
 static DocRepresentation *
-get_docrep(TSVector txt, QueryRepresentation *qr, uint32 *doclen)
+get_docrep(TSVector txt, QueryRepresentation *qr, uint32 *doclen, bool load_idf)
 {
 	QueryItem  *item = GETQUERY(qr->query);
 	WordEntry  *entry,
@@ -1134,6 +1128,8 @@ get_docrep(TSVector txt, QueryRepresentation *qr, uint32 *doclen)
 
 		while (entry - firstentry < nitem)
 		{
+			float4		idf;
+
 			if (entry->haspos)
 			{
 				dimt = POSDATALEN(txt, entry);
@@ -1187,12 +1183,18 @@ get_docrep(TSVector txt, QueryRepresentation *qr, uint32 *doclen)
 
 						}
 					}
+
+					if (load_idf)
+						idf = estimate_idf(STRPTR(txt) + entry->pos, entry->len);
+					else
+						idf = 1.0f;
 				}
 				else
 				{
 					doc[cur].data.item.nitem = doc[cur - 1].data.item.nitem;
 					doc[cur].data.item.item = doc[cur - 1].data.item.item;
 				}
+				doc[cur].idf = idf;
 				doc[cur].pos = WEP_GETPOS(post[j]);
 				doc[cur].wclass = WEP_GETWEIGHT(post[j]);
 				cur++;
@@ -1256,6 +1258,7 @@ calc_score_docr(float4 *arrdata, DocRepresentation *doc, uint32 doclen,
 			/* For rum_tsquery_distance() */
 			else
 				new_cover_key += (int)(uintptr_t)ptr->data.key.item_first;
+			Idf += ptr->idf;
 			ptr++;
 		}
 
@@ -1287,43 +1290,16 @@ calc_score_docr(float4 *arrdata, DocRepresentation *doc, uint32 doclen,
 
 		/* Compute the number of query terms in the cover */
 		for (i = 0; i < qr->length; i++)
-		{
 			if (qr->operandData[i].operandexist)
-			{
-				if (method & RANK_NORM_IDF)
-				{
-					if (!qr->operandIdf[i].idfloaded)
-					{
-						QueryOperand *oper = (QueryOperand *) (GETQUERY(qr->query) + i);
-						qr->operandIdf[i].idf =
-							estimate_idf(
-								GETOPERAND(qr->query) + oper->distance,
-								oper->length
-							);
-						qr->operandIdf[i].idfloaded = true;
-					}
-
-					Idf += qr->operandIdf[i].idf;
-				}
-				else
-				{
-					nitems++;
-				}
-			}
-		}
+				nitems++;
 
 		Cpos = ((double) (ext.end - ext.begin + 1)) / InvSum;
 
+		if (nitems > 0)
+			Cpos *= nitems;
+
 		if (method & RANK_NORM_IDF)
-		{
-			if (Idf >= 1.0)
-				Cpos *= Idf;
-		}
-		else
-		{
-			if (nitems > 0)
-				Cpos *= nitems;
-		}
+			Cpos *= Idf;
 
 		/*
 		 * if doc are big enough then ext.q may be equal to ext.p due to limit
@@ -1408,11 +1384,9 @@ calc_score(float4 *arrdata, TSVector txt, TSQuery query, int method)
 	qr.query = query;
 	qr.map_item_operand = NULL;
 	qr.operandData = palloc0(sizeof(qr.operandData[0]) * query->size);
-	if (method & RANK_NORM_IDF)
-		qr.operandIdf = palloc0(sizeof(qr.operandIdf[0]) * query->size);
 	qr.length = query->size;
 
-	doc = get_docrep(txt, &qr, &doclen);
+	doc = get_docrep(txt, &qr, &doclen, (method & RANK_NORM_IDF) ? true : false);
 	if (!doc)
 	{
 		pfree(qr.operandData);

src/tf_idf.c

@@ -23,8 +23,8 @@
 
 /*
  * FIXME:
- *  * cache IDF
- *  * handle prefix search
+ *  * cache IDF for ts_query (non-prefix search?)
+ *  * calculate IDF from RUM index
  */
 
 /* lookup table type for binary searching through MCELEMs */
@@ -139,6 +139,7 @@ check_tf_idf_source(char **newval, void **extra, GucSource source)
 			if (rel->rd_index->indkey.values[i] == InvalidAttrNumber)
 				exprnum++;
 		}
+		RelationGetIndexExpressions(rel);
 		if (exprType((Node *) list_nth(rel->rd_indexprs, exprnum)) != TSVECTOROID)
 			EXIT_CHECK_TF_IDF_SOURCE("indexed expression should be of tsvector type");
 	}