Add an extra header byte to TOAST-pointer datums to represent their size

explicitly.  This means a TOAST pointer takes 18 bytes instead of 17 --- still
smaller than in 8.2 --- which seems a good tradeoff to ensure we won't have
painted ourselves into a corner if we want to support multiple types of TOAST
pointer later on.  Per discussion with Greg Stark.

Author: tglsfdc

doc/src/sgml/storage.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.19 2007/09/21 21:25:42 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.20 2007/09/30 19:54:57 tgl Exp $ -->
 
 <chapter id="storage">
 
@@ -233,8 +233,8 @@ header, and the remaining bits give the total datum size (including length
 byte) in bytes.  As a special case, if the remaining bits are all zero
 (which would be impossible for a self-inclusive length), the value is a
 pointer to out-of-line data stored in a separate TOAST table.  (The size of
-a TOAST pointer is known a priori, so it doesn't need to be represented in
-the header.)  Values with single-byte headers aren't aligned on any particular
+a TOAST pointer is given in the second byte of the datum.)
+Values with single-byte headers aren't aligned on any particular
 boundary, either.  Lastly, when the highest-order or lowest-order bit is
 clear but the adjacent bit is set, the content of the datum has been
 compressed and must be decompressed before use.  In this case the remaining
@@ -274,8 +274,8 @@ retrieval of the values.  A pointer datum representing an out-of-line
 <acronym>TOAST</> table in which to look and the OID of the specific value
 (its <structfield>chunk_id</>).  For convenience, pointer datums also store the
 logical datum size (original uncompressed data length) and actual stored size
-(different if compression was applied).  Allowing for the varlena header byte,
-the total size of a <acronym>TOAST</> pointer datum is therefore 17 bytes
+(different if compression was applied).  Allowing for the varlena header bytes,
+the total size of a <acronym>TOAST</> pointer datum is therefore 18 bytes
 regardless of the actual size of the represented value.
 </para>
 

src/backend/access/heap/tuptoaster.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.75 2007/09/26 23:29:10 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.76 2007/09/30 19:54:58 tgl Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -52,6 +52,21 @@
 #define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
 	((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
 
+/*
+ * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
+ * into a local "struct varatt_external" toast pointer.  This should be
+ * just a memcpy, but some versions of gcc seem to produce broken code
+ * that assumes the datum contents are aligned.  Introducing an explicit
+ * intermediate "varattrib_1b_e *" variable seems to fix it.
+ */
+#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
+do { \
+	varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
+	Assert(VARSIZE_ANY_EXHDR(attre) == sizeof(toast_pointer)); \
+	memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
+} while (0)
+
+
 static void toast_delete_datum(Relation rel, Datum value);
 static Datum toast_save_datum(Relation rel, Datum value,
 							  bool use_wal, bool use_fsm);
@@ -172,7 +187,7 @@ heap_tuple_untoast_attr_slice(struct varlena *attr,
 	{
 		struct varatt_external toast_pointer;
 
-		memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer));
+		VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
 		/* fast path for non-compressed external datums */
 		if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
@@ -249,7 +264,7 @@ toast_raw_datum_size(Datum value)
 		/* va_rawsize is the size of the original datum -- including header */
 		struct varatt_external toast_pointer;
 
-		memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer));
+		VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 		result = toast_pointer.va_rawsize;
 	}
 	else if (VARATT_IS_COMPRESSED(attr))
@@ -294,7 +309,7 @@ toast_datum_size(Datum value)
 		 */
 		struct varatt_external toast_pointer;
 
-		memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer));
+		VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 		result = toast_pointer.va_extsize;
 	}
 	else if (VARATT_IS_SHORT(attr))
@@ -470,9 +485,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 				VARATT_IS_EXTERNAL(old_value))
 			{
 				if (toast_isnull[i] || !VARATT_IS_EXTERNAL(new_value) ||
-					memcmp(VARDATA_SHORT(old_value),
-						   VARDATA_SHORT(new_value),
-						   sizeof(struct varatt_external)) != 0)
+					memcmp((char *) old_value, (char *) new_value, 
+						   VARSIZE_EXTERNAL(old_value)) != 0)
 				{
 					/*
 					 * The old external stored value isn't needed any more
@@ -1071,7 +1085,7 @@ toast_save_datum(Relation rel, Datum value,
 	Datum		t_values[3];
 	bool		t_isnull[3];
 	CommandId	mycid = GetCurrentCommandId();
-	struct varlena *result;
+	varattrib_pointer *result;
 	struct varatt_external toast_pointer;
 	struct
 	{
@@ -1192,9 +1206,9 @@ toast_save_datum(Relation rel, Datum value,
 	/*
 	 * Create the TOAST pointer value that we'll return
 	 */
-	result = (struct varlena *) palloc(sizeof(varattrib_pointer));
-	SET_VARSIZE_EXTERNAL(result);
-	memcpy(VARDATA_SHORT(result), &toast_pointer, sizeof(toast_pointer));
+	result = (varattrib_pointer *) palloc(sizeof(varattrib_pointer));
+	SET_VARSIZE_EXTERNAL(result, sizeof(varattrib_pointer));
+	memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
 
 	return PointerGetDatum(result);
 }
@@ -1221,8 +1235,7 @@ toast_delete_datum(Relation rel, Datum value)
 		return;
 
 	/* Must copy to access aligned fields */
-	memcpy(&toast_pointer, VARDATA_SHORT(attr),
-		   sizeof(struct varatt_external));
+	VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
 	/*
 	 * Open the toast relation and its index
@@ -1289,8 +1302,7 @@ toast_fetch_datum(struct varlena *attr)
 	int32		chunksize;
 
 	/* Must copy to access aligned fields */
-	memcpy(&toast_pointer, VARDATA_SHORT(attr),
-		   sizeof(struct varatt_external));
+	VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
 	ressize = toast_pointer.va_extsize;
 	numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
@@ -1452,8 +1464,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length)
 	Assert(VARATT_IS_EXTERNAL(attr));
 
 	/* Must copy to access aligned fields */
-	memcpy(&toast_pointer, VARDATA_SHORT(attr),
-		   sizeof(struct varatt_external));
+	VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
 
 	/*
 	 * It's nonsense to fetch slices of a compressed datum -- this isn't lo_*

src/include/catalog/catversion.h

@@ -37,7 +37,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.429 2007/09/25 22:21:55 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.430 2007/09/30 19:54:58 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -53,6 +53,6 @@
  */
 
 /*							yyyymmddN */
-#define CATALOG_VERSION_NO	200709251
+#define CATALOG_VERSION_NO	200709301
 
 #endif

src/include/postgres.h

@@ -10,7 +10,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1995, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/postgres.h,v 1.83 2007/09/27 21:01:59 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/postgres.h,v 1.84 2007/09/30 19:54:58 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -100,12 +100,20 @@ typedef union
 typedef struct
 {
 	uint8		va_header;
-	char		va_data[1];			/* Data or TOAST pointer */
+	char		va_data[1];			/* Data begins here */
 } varattrib_1b;
 
 typedef struct
 {
-	uint8		va_header;
+	uint8		va_header;			/* Always 0x80 or 0x01 */
+	uint8		va_len_1be;			/* Physical length of datum */
+	char		va_data[1];			/* Data (for now always a TOAST pointer) */
+} varattrib_1b_e;
+
+typedef struct
+{
+	uint8		va_header;			/* Always 0x80 or 0x01 */
+	uint8		va_len_1be;			/* Physical length of datum */
 	char		va_data[sizeof(struct varatt_external)];
 } varattrib_pointer;
 
@@ -161,18 +169,18 @@ typedef struct
 	(((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
 #define VARSIZE_1B(PTR) \
 	(((varattrib_1b *) (PTR))->va_header & 0x7F)
-/* Currently there is only one size of toast pointer, but someday maybe not */
 #define VARSIZE_1B_E(PTR) \
-	(sizeof(varattrib_pointer))
+	(((varattrib_1b_e *) (PTR))->va_len_1be)
 
 #define SET_VARSIZE_4B(PTR,len) \
 	(((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
 #define SET_VARSIZE_4B_C(PTR,len) \
 	(((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
 #define SET_VARSIZE_1B(PTR,len) \
 	(((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
-#define SET_VARSIZE_1B_E(PTR) \
-	(((varattrib_1b *) (PTR))->va_header = 0x80)
+#define SET_VARSIZE_1B_E(PTR,len) \
+	(((varattrib_1b_e *) (PTR))->va_header = 0x80, \
+	 ((varattrib_1b_e *) (PTR))->va_len_1be = (len))
 
 #else  /* !WORDS_BIGENDIAN */
 
@@ -194,18 +202,18 @@ typedef struct
 	((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
 #define VARSIZE_1B(PTR) \
 	((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
-/* Currently there is only one size of toast pointer, but someday maybe not */
 #define VARSIZE_1B_E(PTR) \
-	(sizeof(varattrib_pointer))
+	(((varattrib_1b_e *) (PTR))->va_len_1be)
 
 #define SET_VARSIZE_4B(PTR,len) \
 	(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
 #define SET_VARSIZE_4B_C(PTR,len) \
 	(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
 #define SET_VARSIZE_1B(PTR,len) \
 	(((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
-#define SET_VARSIZE_1B_E(PTR) \
-	(((varattrib_1b *) (PTR))->va_header = 0x01)
+#define SET_VARSIZE_1B_E(PTR,len) \
+	(((varattrib_1b_e *) (PTR))->va_header = 0x01, \
+	 ((varattrib_1b_e *) (PTR))->va_len_1be = (len))
 
 #endif /* WORDS_BIGENDIAN */
 
@@ -220,6 +228,7 @@ typedef struct
 #define VARDATA_4B(PTR)		(((varattrib_4b *) (PTR))->va_4byte.va_data)
 #define VARDATA_4B_C(PTR)	(((varattrib_4b *) (PTR))->va_compressed.va_data)
 #define VARDATA_1B(PTR)		(((varattrib_1b *) (PTR))->va_data)
+#define VARDATA_1B_E(PTR)	(((varattrib_1b_e *) (PTR))->va_data)
 
 #define VARRAWSIZE_4B_C(PTR) \
 	(((varattrib_4b *) (PTR))->va_compressed.va_rawsize)
@@ -249,6 +258,7 @@ typedef struct
 #define VARDATA_SHORT(PTR)					VARDATA_1B(PTR)
 
 #define VARSIZE_EXTERNAL(PTR)				VARSIZE_1B_E(PTR)
+#define VARDATA_EXTERNAL(PTR)				VARDATA_1B_E(PTR)
 
 #define VARATT_IS_COMPRESSED(PTR)			VARATT_IS_4B_C(PTR)
 #define VARATT_IS_EXTERNAL(PTR)				VARATT_IS_1B_E(PTR)
@@ -258,15 +268,15 @@ typedef struct
 #define SET_VARSIZE(PTR, len)				SET_VARSIZE_4B(PTR, len)
 #define SET_VARSIZE_SHORT(PTR, len)			SET_VARSIZE_1B(PTR, len)
 #define SET_VARSIZE_COMPRESSED(PTR, len)	SET_VARSIZE_4B_C(PTR, len)
-#define SET_VARSIZE_EXTERNAL(PTR)			SET_VARSIZE_1B_E(PTR)
+#define SET_VARSIZE_EXTERNAL(PTR, len)		SET_VARSIZE_1B_E(PTR, len)
 
 #define VARSIZE_ANY(PTR) \
 	(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \
 	 (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \
 	  VARSIZE_4B(PTR)))
 
 #define VARSIZE_ANY_EXHDR(PTR) \
-	(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-1 : \
+	(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-2 : \
 	 (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-1 : \
 	  VARSIZE_4B(PTR)-4))