From f422c8172fb2e70f31e06a189b875f01eb2d3ba8 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Sat, 12 Jun 1999 14:05:41 +0000 Subject: [PATCH] I don't like last minute patches before the final freeze, but I believe that this one could be useful for people experiencing out-of-memory crashes while executing queries which retrieve or use a very large number of tuples. The problem happens when storage is allocated for functions results used in a large query, for example: select upper(name) from big_table; select big_table.array[1] from big_table; select count(upper(name)) from big_table; This patch is a dirty hack that fixes the out-of-memory problem for the most common cases, like the above ones. It is not the final solution for the problem but it can work for some people, so I'm posting it. The patch should be safe because all changes are under #ifdef. Furthermore the feature can be enabled or disabled at runtime by the `free_tuple_memory' options in the pg_options file. The option is disabled by default and must be explicitly enabled at runtime to have any effect. To enable the patch add the follwing line to Makefile.custom: CUSTOM_COPT += -DFREE_TUPLE_MEMORY To enable the option at runtime add the following line to pg_option: free_tuple_memory=1 Massimo --- src/backend/access/common/heaptuple.c | 27 +++++++++++++-- src/backend/executor/nodeAgg.c | 48 ++++++++++++++++++++++++++- src/backend/utils/misc/trace.c | 11 +++--- src/backend/utils/mmgr/portalmem.c | 18 ++++++++++ src/include/utils/portal.h | 4 +++ src/include/utils/trace.h | 11 +++--- 6 files changed, 108 insertions(+), 11 deletions(-) diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index b20ad6dc5c..1a27b8625e 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -27,6 +27,11 @@ #include #include +#ifdef FREE_TUPLE_MEMORY +#include +#include +#endif + #ifndef HAVE_MEMMOVE #include #else @@ -93,6 +98,9 @@ DataFill(char *data, int i; int numberOfAttributes = tupleDesc->natts; Form_pg_attribute *att = tupleDesc->attrs; +#ifdef FREE_TUPLE_MEMORY + bool free_tuple_memory = pg_options[OPT_FREE_TUPLE_MEMORY]; +#endif if (bit != NULL) { @@ -131,6 +139,14 @@ DataFill(char *data, *infomask |= HEAP_HASVARLENA; data_length = VARSIZE(DatumGetPointer(value[i])); memmove(data, DatumGetPointer(value[i]), data_length); +#ifdef FREE_TUPLE_MEMORY + /* try to pfree value[i] - dz */ + if (free_tuple_memory && + PortalHeapMemoryIsValid(CurrentMemoryContext, + (Pointer) value[i])) { + pfree(value[i]); + } +#endif break; case sizeof(char): *data = att[i]->attbyval ? @@ -147,8 +163,15 @@ DataFill(char *data, *((int32 *) value[i])); break; default: - memmove(data, DatumGetPointer(value[i]), - att[i]->attlen); + memmove(data, DatumGetPointer(value[i]), att[i]->attlen); +#ifdef FREE_TUPLE_MEMORY + /* try to pfree value[i] - dz */ + if (free_tuple_memory && + PortalHeapMemoryIsValid(CurrentMemoryContext, + (Pointer) value[i])) { + pfree(value[i]); + } +#endif break; } data = (char *) att_addlength((long) data, att[i]->attlen, value[i]); diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index a47a9ad549..9d0f4ea103 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -31,6 +31,11 @@ #include "utils/syscache.h" #include "optimizer/clauses.h" +#ifdef FREE_TUPLE_MEMORY +#include +#include +#endif + /* * AggFuncInfo - * keeps the transition functions information around @@ -113,7 +118,9 @@ ExecAgg(Agg *node) isNull1 = FALSE, isNull2 = FALSE; bool qual_result; - +#ifdef FREE_TUPLE_MEMORY + bool free_tuple_memory = pg_options[OPT_FREE_TUPLE_MEMORY]; +#endif /* --------------------- * get state info from node @@ -241,6 +248,10 @@ ExecAgg(Agg *node) for (;;) { TupleTableSlot *outerslot; +#ifdef FREE_TUPLE_MEMORY + Oid valueType; + bool isByValue = 0; +#endif isNull = isNull1 = isNull2 = 0; outerslot = ExecProcNode(outerPlan, (Plan *) node); @@ -293,6 +304,31 @@ ExecAgg(Agg *node) newVal = ExecEvalExpr(aggref->target, econtext, &isNull, &isDone); } +#ifdef FREE_TUPLE_MEMORY + if (free_tuple_memory) { + switch (nodeTag(aggref->target)) { + case T_Const: + isByValue = ((Const*) (aggref->target))->constbyval; + break; + case T_Var: + valueType = ((Var*) (aggref->target))->vartype; + isByValue = typeByVal(typeidType(valueType)); + break; + case T_Array: + isByValue = ((Array*)(aggref->target))->arrayelembyval; + break; + case T_ArrayRef: + isByValue =((ArrayRef*)(aggref->target))->refelembyval; + break; + case T_Expr: + valueType = ((Expr*) (aggref->target))->typeOid; + isByValue = typeByVal(typeidType(valueType)); + break; + default: + break; + } + } +#endif if (isNull && !aggref->usenulls) continue; /* ignore this tuple for this agg */ @@ -353,6 +389,16 @@ ExecAgg(Agg *node) (FmgrValues *) args, &isNull2); Assert(!isNull2); } + +#ifdef FREE_TUPLE_MEMORY + /* try to pfree newVal if not isByValue - dz */ + if (free_tuple_memory && !isByValue && + PortalHeapMemoryIsValid(CurrentMemoryContext, + (Pointer) newVal)) + { + pfree(newVal); + } +#endif } /* diff --git a/src/backend/utils/misc/trace.c b/src/backend/utils/misc/trace.c index 9676ca570a..5a18a21390 100644 --- a/src/backend/utils/misc/trace.c +++ b/src/backend/utils/misc/trace.c @@ -73,6 +73,9 @@ static char *opt_names[] = { "syslog", /* use syslog for error messages */ "hostlookup", /* enable hostname lookup in ps_status */ "showportnumber", /* show port number in ps_status */ +#ifdef FREE_TUPLE_MEMORY + "free_tuple_memory", /* try to pfree memory for each tuple */ +#endif /* NUM_PG_OPTIONS */ /* must be the last item of enum */ }; @@ -404,9 +407,9 @@ read_pg_options(SIGNAL_ARGS) } /* - * Local variables: - * tab-width: 4 - * c-indent-level: 4 - * c-basic-offset: 4 + * Local Variables: + * tab-width: 4 + * c-indent-level: 4 + * c-basic-offset: 4 * End: */ diff --git a/src/backend/utils/mmgr/portalmem.c b/src/backend/utils/mmgr/portalmem.c index 86c4ff6a2a..90d2ff7685 100644 --- a/src/backend/utils/mmgr/portalmem.c +++ b/src/backend/utils/mmgr/portalmem.c @@ -289,6 +289,24 @@ PortalHeapMemoryFree(PortalHeapMemory this, } } +#ifdef FREE_TUPLE_MEMORY +/* + * PortalHeapMemoryIsValid -- + * + * Check if a pointer is allocated in a memory context. + * + */ +bool +PortalHeapMemoryIsValid(MemoryContext context, Pointer pointer) +{ + HeapMemoryBlock block = HEAPMEMBLOCK((PortalHeapMemory) context); + + AssertState(PointerIsValid(block)); + + return (AllocSetContains(&block->setData, pointer)); +} +#endif + /* ---------------- * PortalHeapMemoryRealloc * ---------------- diff --git a/src/include/utils/portal.h b/src/include/utils/portal.h index b26ddf525d..f5ada649de 100644 --- a/src/include/utils/portal.h +++ b/src/include/utils/portal.h @@ -80,6 +80,10 @@ extern void EndPortalAllocMode(void); extern PortalVariableMemory PortalGetVariableMemory(Portal portal); extern PortalHeapMemory PortalGetHeapMemory(Portal portal); +#ifdef FREE_TUPLE_MEMORY +bool PortalHeapMemoryIsValid(MemoryContext context, Pointer pointer); +#endif + /* estimate of the maximum number of open portals a user would have, * used in initially sizing the PortalHashTable in EnablePortalManager() */ diff --git a/src/include/utils/trace.h b/src/include/utils/trace.h index 3174026ca5..1affd968b7 100644 --- a/src/include/utils/trace.h +++ b/src/include/utils/trace.h @@ -64,6 +64,9 @@ enum pg_option_enum { OPT_SYSLOG, /* use syslog for error messages */ OPT_HOSTLOOKUP, /* enable hostname lookup in ps_status */ OPT_SHOWPORTNUMBER, /* show port number in ps_status */ +#ifdef FREE_TUPLE_MEMORY + OPT_FREE_TUPLE_MEMORY, /* try to pfree memory for each tuple */ +#endif NUM_PG_OPTIONS /* must be the last item of enum */ }; @@ -83,9 +86,9 @@ extern int pg_options[NUM_PG_OPTIONS]; #endif /* TRACE_H */ /* - * Local variables: - * tab-width: 4 - * c-indent-level: 4 - * c-basic-offset: 4 + * Local Variables: + * tab-width: 4 + * c-indent-level: 4 + * c-basic-offset: 4 * End: */ -- 2.39.5