Retrofit hashtable and shared-mem-size-estimation bug fixes

into REL6_4.

Author: tglsfdc

src/backend/storage/buffer/buf_init.c

@@ -7,7 +7,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/buffer/buf_init.c,v 1.19 1998/09/01 04:31:39 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/buffer/buf_init.c,v 1.19.2.1 1999/03/07 02:01:08 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -33,7 +33,6 @@
 #include "storage/lmgr.h"
 #include "miscadmin.h"
 #include "utils/builtins.h"
-#include "utils/dynahash.h"
 #include "utils/hsearch.h"
 #include "utils/memutils.h"
 #include "executor/execdebug.h" /* for NDirectFileRead */
@@ -270,21 +269,11 @@ int
 BufferShmemSize()
 {
 	int			size = 0;
-	int			nbuckets;
-	int			nsegs;
-	int			tmp;
-
-	nbuckets = 1 << (int) my_log2((NBuffers - 1) / DEF_FFACTOR + 1);
-	nsegs = 1 << (int) my_log2((nbuckets - 1) / DEF_SEGSIZE + 1);
-
-	/* size of shmem index table */
-	size += MAXALIGN(my_log2(SHMEM_INDEX_SIZE) * sizeof(void *));		/* HTAB->dir */
-	size += MAXALIGN(sizeof(HHDR));		/* HTAB->hctl */
-	size += MAXALIGN(DEF_SEGSIZE * sizeof(SEGMENT));
-	size += BUCKET_ALLOC_INCR *
-		(MAXALIGN(sizeof(BUCKET_INDEX)) +
-		 MAXALIGN(SHMEM_INDEX_KEYSIZE) +
-		 MAXALIGN(SHMEM_INDEX_DATASIZE));
+
+	/* size of shmem index hash table */
+	size += hash_estimate_size(SHMEM_INDEX_SIZE,
+							   SHMEM_INDEX_KEYSIZE,
+							   SHMEM_INDEX_DATASIZE);
 
 	/* size of buffer descriptors */
 	size += MAXALIGN((NBuffers + 1) * sizeof(BufferDesc));
@@ -293,17 +282,13 @@ BufferShmemSize()
 	size += NBuffers * MAXALIGN(BLCKSZ);
 
 	/* size of buffer hash table */
-	size += MAXALIGN(my_log2(NBuffers) * sizeof(void *));		/* HTAB->dir */
-	size += MAXALIGN(sizeof(HHDR));		/* HTAB->hctl */
-	size += nsegs * MAXALIGN(DEF_SEGSIZE * sizeof(SEGMENT));
-	tmp = (int) ceil((double) NBuffers / BUCKET_ALLOC_INCR);
-	size += tmp * BUCKET_ALLOC_INCR *
-		(MAXALIGN(sizeof(BUCKET_INDEX)) +
-		 MAXALIGN(sizeof(BufferTag)) +
-		 MAXALIGN(sizeof(Buffer)));
+	size += hash_estimate_size(NBuffers,
+							   sizeof(BufferTag),
+							   sizeof(Buffer));
 
 #ifdef BMTRACE
 	size += (BMT_LIMIT * sizeof(bmtrace)) + sizeof(long);
 #endif
+
 	return size;
 }

src/backend/storage/ipc/ipci.c

@@ -7,7 +7,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/ipci.c,v 1.16 1998/09/01 03:25:10 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/ipci.c,v 1.16.2.1 1999/03/07 02:00:46 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -71,11 +71,17 @@ CreateSharedMemoryAndSemaphores(IPCKey key)
 	 * ----------------
 	 */
 	CreateSpinlocks(IPCKeyGetSpinLockSemaphoreKey(key));
-	size = BufferShmemSize() + LockShmemSize();
 
+	/*
+	 * Size of the primary shared-memory block is estimated via
+	 * moderately-accurate estimates for the big hogs, plus 100K for
+	 * the stuff that's too small to bother with estimating.
+	 */
+	size = BufferShmemSize() + LockShmemSize();
 #ifdef STABLE_MEMORY_STORAGE
 	size += MMShmemSize();
 #endif
+	size += 100000;
 
 	if (DebugLvl > 1)
 	{
@@ -113,8 +119,6 @@ CreateSharedMemoryAndSemaphores(IPCKey key)
 void
 AttachSharedMemoryAndSemaphores(IPCKey key)
 {
-	int			size;
-
 	/* ----------------
 	 *	create rather than attach if using private key
 	 * ----------------
@@ -136,8 +140,7 @@ AttachSharedMemoryAndSemaphores(IPCKey key)
 	 *	attach the buffer manager buffer pool (and semaphore)
 	 * ----------------
 	 */
-	size = BufferShmemSize() + LockShmemSize();
-	InitShmem(key, size);
+	InitShmem(key, 0);
 	InitBufferPool(key);
 
 	/* ----------------

src/backend/storage/ipc/shmem.c

@@ -7,7 +7,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/shmem.c,v 1.31 1998/09/01 04:31:49 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/shmem.c,v 1.31.2.1 1999/03/07 02:00:46 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -215,7 +215,7 @@ InitShmem(unsigned int key, unsigned int size)
 	/* create OR attach to the shared memory shmem index */
 	info.keysize = SHMEM_INDEX_KEYSIZE;
 	info.datasize = SHMEM_INDEX_DATASIZE;
-	hash_flags = (HASH_ELEM);
+	hash_flags = HASH_ELEM;
 
 	/* This will acquire the shmem index lock, but not release it. */
 	ShmemIndex = ShmemInitHash("ShmemIndex",
@@ -340,27 +340,29 @@ ShmemIsValid(unsigned long addr)
  */
 HTAB *
 ShmemInitHash(char *name,		/* table string name for shmem index */
-			  long init_size,	/* initial size */
-			  long max_size,	/* max size of the table */
+			  long init_size,	/* initial table size */
+			  long max_size,	/* max size of the table (NOT USED) */
 			  HASHCTL *infoP,	/* info about key and bucket size */
 			  int hash_flags)	/* info about infoP */
 {
 	bool		found;
 	long	   *location;
 
 	/*
-	 * shared memory hash tables have a fixed max size so that the control
-	 * structures don't try to grow.  The segbase is for calculating
-	 * pointer values.	The shared memory allocator must be specified.
+	 * Hash tables allocated in shared memory have a fixed directory;
+	 * it can't grow or other backends wouldn't be able to find it.
+	 * The segbase is for calculating pointer values.
+	 * The shared memory allocator must be specified too.
 	 */
+	infoP->dsize = infoP->max_dsize = DEF_DIRSIZE;
 	infoP->segbase = (long *) ShmemBase;
 	infoP->alloc = ShmemAlloc;
-	infoP->max_size = max_size;
-	hash_flags |= HASH_SHARED_MEM;
+	hash_flags |= HASH_SHARED_MEM | HASH_DIRSIZE;
 
 	/* look it up in the shmem index */
-	location =
-		ShmemInitStruct(name, my_log2(max_size) + sizeof(HHDR), &found);
+	location = ShmemInitStruct(name,
+							   sizeof(HHDR) + DEF_DIRSIZE * sizeof(SEG_OFFSET),
+							   &found);
 
 	/*
 	 * shmem index is corrupted.	Let someone else give the error
@@ -376,13 +378,11 @@ ShmemInitHash(char *name,		/* table string name for shmem index */
 	if (found)
 		hash_flags |= HASH_ATTACH;
 
-	/* these structures were allocated or bound in ShmemInitStruct */
-	/* control information and parameters */
+	/* Now provide the header and directory pointers */
 	infoP->hctl = (long *) location;
-	/* directory for hash lookup */
-	infoP->dir = (long *) (location + sizeof(HHDR));
+	infoP->dir = (long *) (((char*) location) + sizeof(HHDR));
 
-	return hash_create(init_size, infoP, hash_flags);;
+	return hash_create(init_size, infoP, hash_flags);
 }
 
 /*

src/backend/storage/lmgr/lock.c

@@ -7,7 +7,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/lmgr/lock.c,v 1.38 1998/10/08 18:29:57 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/lmgr/lock.c,v 1.38.2.1 1999/03/07 02:00:49 tgl Exp $
  *
  * NOTES
  *	  Outside modules can create a lock table and acquire/release
@@ -42,7 +42,6 @@
 #include "storage/spin.h"
 #include "storage/proc.h"
 #include "storage/lock.h"
-#include "utils/dynahash.h"
 #include "utils/hsearch.h"
 #include "utils/memutils.h"
 #include "utils/palloc.h"
@@ -340,8 +339,8 @@ LockMethodTableInit(char *tabName,
 	 * to find the different locks.
 	 * ----------------------
 	 */
-	info.keysize = sizeof(LOCKTAG);
-	info.datasize = sizeof(LOCK);
+	info.keysize = SHMEM_LOCKTAB_KEYSIZE;
+	info.datasize = SHMEM_LOCKTAB_DATASIZE;
 	info.hash = tag_hash;
 	hash_flags = (HASH_ELEM | HASH_FUNCTION);
 
@@ -362,8 +361,8 @@ LockMethodTableInit(char *tabName,
 	 * the same lock, additional information must be saved (locks per tx).
 	 * -------------------------
 	 */
-	info.keysize = XID_TAGSIZE;
-	info.datasize = sizeof(XIDLookupEnt);
+	info.keysize = SHMEM_XIDTAB_KEYSIZE;
+	info.datasize = SHMEM_XIDTAB_DATASIZE;
 	info.hash = tag_hash;
 	hash_flags = (HASH_ELEM | HASH_FUNCTION);
 
@@ -1491,35 +1490,26 @@ int
 LockShmemSize()
 {
 	int			size = 0;
-	int			nLockBuckets,
-				nLockSegs;
-	int			nXidBuckets,
-				nXidSegs;
 
-	nLockBuckets = 1 << (int) my_log2((NLOCKENTS - 1) / DEF_FFACTOR + 1);
-	nLockSegs = 1 << (int) my_log2((nLockBuckets - 1) / DEF_SEGSIZE + 1);
+	size += MAXALIGN(sizeof(PROC_HDR)); /* ProcGlobal */
+	size += MAXALIGN(MaxBackendId * sizeof(PROC)); /* each MyProc */
+	size += MAXALIGN(MaxBackendId * sizeof(LOCKMETHODCTL));		/* each
+																 * lockMethodTable->ctl */
 
-	nXidBuckets = 1 << (int) my_log2((NLOCKS_PER_XACT - 1) / DEF_FFACTOR + 1);
-	nXidSegs = 1 << (int) my_log2((nLockBuckets - 1) / DEF_SEGSIZE + 1);
+	/* lockHash table */
+	size += hash_estimate_size(NLOCKENTS,
+							   SHMEM_LOCKTAB_KEYSIZE,
+							   SHMEM_LOCKTAB_DATASIZE);
 
-	size += MAXALIGN(NBACKENDS * sizeof(PROC)); /* each MyProc */
-	size += MAXALIGN(NBACKENDS * sizeof(LOCKMETHODCTL));		/* each
-																 * lockMethodTable->ctl */
-	size += MAXALIGN(sizeof(PROC_HDR)); /* ProcGlobal */
+	/* xidHash table */
+	size += hash_estimate_size(MaxBackendId,
+							   SHMEM_XIDTAB_KEYSIZE,
+							   SHMEM_XIDTAB_DATASIZE);
 
-	size += MAXALIGN(my_log2(NLOCKENTS) * sizeof(void *));
-	size += MAXALIGN(sizeof(HHDR));
-	size += nLockSegs * MAXALIGN(DEF_SEGSIZE * sizeof(SEGMENT));
-	size += NLOCKENTS *			/* XXX not multiple of BUCKET_ALLOC_INCR? */
-		(MAXALIGN(sizeof(BUCKET_INDEX)) +
-		 MAXALIGN(sizeof(LOCK)));		/* contains hash key */
-
-	size += MAXALIGN(my_log2(NBACKENDS) * sizeof(void *));
-	size += MAXALIGN(sizeof(HHDR));
-	size += nXidSegs * MAXALIGN(DEF_SEGSIZE * sizeof(SEGMENT));
-	size += NBACKENDS *			/* XXX not multiple of BUCKET_ALLOC_INCR? */
-		(MAXALIGN(sizeof(BUCKET_INDEX)) +
-		 MAXALIGN(sizeof(XIDLookupEnt)));		/* contains hash key */
+	/* Since the lockHash entry count above is only an estimate,
+	 * add 10% safety margin.
+	 */
+	size += size / 10;
 
 	return size;
 }

src/backend/storage/smgr/mm.c

@@ -10,7 +10,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/smgr/Attic/mm.c,v 1.12 1998/09/01 04:32:07 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/smgr/Attic/mm.c,v 1.12.2.1 1999/03/07 02:00:52 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -25,7 +25,6 @@
 #include "storage/shmem.h"
 #include "storage/spin.h"
 
-#include "utils/dynahash.h"
 #include "utils/hsearch.h"
 #include "utils/rel.h"
 #include "utils/memutils.h"
@@ -111,7 +110,7 @@ mminit()
 	}
 
 	info.keysize = sizeof(MMCacheTag);
-	info.datasize = sizeof(int);
+	info.datasize = sizeof(MMHashEntry) - sizeof(MMCacheTag);
 	info.hash = tag_hash;
 
 	MMCacheHT = (HTAB *) ShmemInitHash("Main memory store HT",
@@ -125,7 +124,7 @@ mminit()
 	}
 
 	info.keysize = sizeof(MMRelTag);
-	info.datasize = sizeof(int);
+	info.datasize = sizeof(MMRelHashEntry) - sizeof(MMRelTag);
 	info.hash = tag_hash;
 
 	MMRelCacheHT = (HTAB *) ShmemInitHash("Main memory rel HT",
@@ -565,36 +564,20 @@ int
 MMShmemSize()
 {
 	int			size = 0;
-	int			nbuckets;
-	int			nsegs;
-	int			tmp;
 
 	/*
 	 * first compute space occupied by the (dbid,relid,blkno) hash table
 	 */
-
-	nbuckets = 1 << (int) my_log2((MMNBUFFERS - 1) / DEF_FFACTOR + 1);
-	nsegs = 1 << (int) my_log2((nbuckets - 1) / DEF_SEGSIZE + 1);
-
-	size += MAXALIGN(my_log2(MMNBUFFERS) * sizeof(void *));
-	size += MAXALIGN(sizeof(HHDR));
-	size += nsegs * MAXALIGN(DEF_SEGSIZE * sizeof(SEGMENT));
-	tmp = (int) ceil((double) MMNBUFFERS / BUCKET_ALLOC_INCR);
-	size += tmp * BUCKET_ALLOC_INCR *
-		(MAXALIGN(sizeof(BUCKET_INDEX)) +
-		 MAXALIGN(sizeof(MMHashEntry)));		/* contains hash key */
+	size += hash_estimate_size(MMNBUFFERS,
+				   0, /* MMHashEntry includes key */
+				   sizeof(MMHashEntry));
 
 	/*
 	 * now do the same for the rel hash table
 	 */
-
-	size += MAXALIGN(my_log2(MMNRELATIONS) * sizeof(void *));
-	size += MAXALIGN(sizeof(HHDR));
-	size += nsegs * MAXALIGN(DEF_SEGSIZE * sizeof(SEGMENT));
-	tmp = (int) ceil((double) MMNRELATIONS / BUCKET_ALLOC_INCR);
-	size += tmp * BUCKET_ALLOC_INCR *
-		(MAXALIGN(sizeof(BUCKET_INDEX)) +
-		 MAXALIGN(sizeof(MMRelHashEntry)));		/* contains hash key */
+	size += hash_estimate_size(MMNRELATIONS,
+				   0, /* MMRelHashEntry includes key */
+				   sizeof(MMRelHashEntry));
 
 	/*
 	 * finally, add in the memory block we use directly