Reject ANALYZE commands during VACUUM FULL or another ANALYZE.

vacuum()'s static variable handling makes it non-reentrant; an ensuing
null pointer deference crashed the backend.  Back-patch to 9.0 (all
supported versions).

Author: nmisch

src/backend/commands/vacuum.c

@@ -105,6 +105,7 @@ vacuum(VacuumStmt *vacstmt, Oid relid, bool do_toast,
 	volatile bool in_outer_xact,
 				use_own_xacts;
 	List	   *relations;
+	static bool in_vacuum = false;
 
 	/* sanity checks on options */
 	Assert(vacstmt->options & (VACOPT_VACUUM | VACOPT_ANALYZE));
@@ -130,6 +131,14 @@ vacuum(VacuumStmt *vacstmt, Oid relid, bool do_toast,
 	else
 		in_outer_xact = IsInTransactionChain(isTopLevel);
 
+	/*
+	 * Due to static variables vac_context, anl_context and vac_strategy,
+	 * vacuum() is not reentrant.  This matters when VACUUM FULL or ANALYZE
+	 * calls a hostile index expression that itself calls ANALYZE.
+	 */
+	if (in_vacuum)
+		elog(ERROR, "%s cannot be executed from VACUUM or ANALYZE", stmttype);
+
 	/*
 	 * Send info about dead objects to the statistics collector, unless we are
 	 * in autovacuum --- autovacuum.c does this for itself.
@@ -222,6 +231,7 @@ vacuum(VacuumStmt *vacstmt, Oid relid, bool do_toast,
 	{
 		ListCell   *cur;
 
+		in_vacuum = true;
 		VacuumCostActive = (VacuumCostDelay > 0);
 		VacuumCostBalance = 0;
 		VacuumPageHit = 0;
@@ -266,13 +276,13 @@ vacuum(VacuumStmt *vacstmt, Oid relid, bool do_toast,
 	}
 	PG_CATCH();
 	{
-		/* Make sure cost accounting is turned off after error */
+		in_vacuum = false;
 		VacuumCostActive = false;
 		PG_RE_THROW();
 	}
 	PG_END_TRY();
 
-	/* Turn off vacuum cost accounting */
+	in_vacuum = false;
 	VacuumCostActive = false;
 
 	/*

src/test/regress/expected/vacuum.out

@@ -60,12 +60,24 @@ VACUUM (FULL, FREEZE) vactst;
 VACUUM (ANALYZE, FULL) vactst;
 CREATE TABLE vaccluster (i INT PRIMARY KEY);
 ALTER TABLE vaccluster CLUSTER ON vaccluster_pkey;
-INSERT INTO vaccluster SELECT * FROM vactst;
 CLUSTER vaccluster;
+CREATE FUNCTION do_analyze() RETURNS VOID VOLATILE LANGUAGE SQL
+	AS 'ANALYZE pg_am';
+CREATE FUNCTION wrap_do_analyze(c INT) RETURNS INT IMMUTABLE LANGUAGE SQL
+	AS 'SELECT $1 FROM do_analyze()';
+CREATE INDEX ON vactst(wrap_do_analyze(i));
+INSERT INTO vactst VALUES (1), (2);
+ANALYZE vactst;
+ERROR:  ANALYZE cannot be executed from VACUUM or ANALYZE
+CONTEXT:  SQL function "do_analyze" statement 1
+SQL function "wrap_do_analyze" statement 1
 VACUUM FULL pg_am;
 VACUUM FULL pg_class;
 VACUUM FULL pg_database;
 VACUUM FULL vaccluster;
 VACUUM FULL vactst;
+ERROR:  ANALYZE cannot be executed from VACUUM or ANALYZE
+CONTEXT:  SQL function "do_analyze" statement 1
+SQL function "wrap_do_analyze" statement 1
 DROP TABLE vaccluster;
 DROP TABLE vactst;

src/test/regress/sql/vacuum.sql

@@ -44,9 +44,16 @@ VACUUM (ANALYZE, FULL) vactst;
 
 CREATE TABLE vaccluster (i INT PRIMARY KEY);
 ALTER TABLE vaccluster CLUSTER ON vaccluster_pkey;
-INSERT INTO vaccluster SELECT * FROM vactst;
 CLUSTER vaccluster;
 
+CREATE FUNCTION do_analyze() RETURNS VOID VOLATILE LANGUAGE SQL
+	AS 'ANALYZE pg_am';
+CREATE FUNCTION wrap_do_analyze(c INT) RETURNS INT IMMUTABLE LANGUAGE SQL
+	AS 'SELECT $1 FROM do_analyze()';
+CREATE INDEX ON vactst(wrap_do_analyze(i));
+INSERT INTO vactst VALUES (1), (2);
+ANALYZE vactst;
+
 VACUUM FULL pg_am;
 VACUUM FULL pg_class;
 VACUUM FULL pg_database;