Skip to content

Commit af1a614

Browse files
adunstanadunstan
committed
Allow the low level COPY routines to read arbitrary numbers of fields.
This doesn't involve any user-visible change in behavior, but will be useful when the COPY routines are exposed to allow their use by Foreign Data Wrapper routines, which will be able to use these routines to read irregular CSV files, for example.
1 parent 95e42a2 commit af1a614

File tree

1 file changed

+57
-36
lines changed

1 file changed

+57
-36
lines changed

‎src/backend/commands/copy.c‎

Lines changed: 57 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,11 @@ typedef struct CopyStateData
141141
*/
142142
StringInfoData attribute_buf;
143143

144+
/* field raw data pointers found by COPY FROM */
145+
146+
int max_fields;
147+
char ** raw_fields;
148+
144149
/*
145150
* Similarly, line_buf holds the whole input line being processed. The
146151
* input cycle is first to read the whole line into line_buf, convert it
@@ -250,10 +255,8 @@ static void CopyOneRowTo(CopyState cstate, Oid tupleOid,
250255
static void CopyFrom(CopyState cstate);
251256
static bool CopyReadLine(CopyState cstate);
252257
static bool CopyReadLineText(CopyState cstate);
253-
static int CopyReadAttributesText(CopyState cstate, int maxfields,
254-
char **fieldvals);
255-
static int CopyReadAttributesCSV(CopyState cstate, int maxfields,
256-
char **fieldvals);
258+
static int CopyReadAttributesText(CopyState cstate);
259+
static int CopyReadAttributesCSV(CopyState cstate);
257260
static Datum CopyReadBinaryAttribute(CopyState cstate,
258261
int column_no, FmgrInfo *flinfo,
259262
Oid typioparam, int32 typmod,
@@ -1921,7 +1924,11 @@ CopyFrom(CopyState cstate)
19211924

19221925
/* create workspace for CopyReadAttributes results */
19231926
nfields = file_has_oids ? (attr_count + 1) : attr_count;
1924-
field_strings = (char **) palloc(nfields * sizeof(char *));
1927+
if (! cstate->binary)
1928+
{
1929+
cstate->max_fields = nfields;
1930+
cstate->raw_fields = (char **) palloc(nfields * sizeof(char *));
1931+
}
19251932

19261933
/* Initialize state variables */
19271934
cstate->fe_eof = false;
@@ -1985,10 +1992,18 @@ CopyFrom(CopyState cstate)
19851992

19861993
/* Parse the line into de-escaped field values */
19871994
if (cstate->csv_mode)
1988-
fldct = CopyReadAttributesCSV(cstate, nfields, field_strings);
1995+
fldct = CopyReadAttributesCSV(cstate);
19891996
else
1990-
fldct = CopyReadAttributesText(cstate, nfields, field_strings);
1997+
fldct = CopyReadAttributesText(cstate);
1998+
1999+
/* check for overflowing fields */
2000+
if (nfields > 0 && fldct > nfields)
2001+
ereport(ERROR,
2002+
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2003+
errmsg("extra data after last expected column")));
2004+
19912005
fieldno = 0;
2006+
field_strings = cstate->raw_fields;
19922007

19932008
/* Read the OID field if present */
19942009
if (file_has_oids)
@@ -2218,7 +2233,8 @@ CopyFrom(CopyState cstate)
22182233

22192234
pfree(values);
22202235
pfree(nulls);
2221-
pfree(field_strings);
2236+
if (! cstate->binary)
2237+
pfree(cstate->raw_fields);
22222238

22232239
pfree(in_functions);
22242240
pfree(typioparams);
@@ -2717,21 +2733,22 @@ GetDecimalFromHex(char hex)
27172733
* performing de-escaping as needed.
27182734
*
27192735
* The input is in line_buf. We use attribute_buf to hold the result
2720-
* strings. fieldvals[k] is set to point to the k'th attribute string,
2721-
* or NULL when the input matches the null marker string. (Note that the
2722-
* caller cannot check for nulls since the returned string would be the
2723-
* post-de-escaping equivalent, which may look the same as some valid data
2724-
* string.)
2736+
* strings. cstate->raw_fields[k] is set to point to the k'th attribute
2737+
* string, or NULL when the input matches the null marker string.
2738+
* This array is expanded as necessary.
2739+
*
2740+
* (Note that the caller cannot check for nulls since the returned
2741+
* string would be the post-de-escaping equivalent, which may look
2742+
* the same as some valid data string.)
27252743
*
27262744
* delim is the column delimiter string (must be just one byte for now).
27272745
* null_print is the null marker string. Note that this is compared to
27282746
* the pre-de-escaped input string.
27292747
*
2730-
* The return value is the number of fields actually read. (We error out
2731-
* if this would exceed maxfields, which is the length of fieldvals[].)
2748+
* The return value is the number of fields actually read.
27322749
*/
27332750
static int
2734-
CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
2751+
CopyReadAttributesText(CopyState cstate)
27352752
{
27362753
char delimc = cstate->delim[0];
27372754
int fieldno;
@@ -2743,7 +2760,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
27432760
* We need a special case for zero-column tables: check that the input
27442761
* line is empty, and return.
27452762
*/
2746-
if (maxfields <= 0)
2763+
if (cstate->max_fields <= 0)
27472764
{
27482765
if (cstate->line_buf.len != 0)
27492766
ereport(ERROR,
@@ -2759,7 +2776,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
27592776
* data line, so we can just force attribute_buf to be large enough and
27602777
* then transfer data without any checks for enough space. We need to do
27612778
* it this way because enlarging attribute_buf mid-stream would invalidate
2762-
* pointers already stored into fieldvals[].
2779+
* pointers already stored into cstate->raw_fields[].
27632780
*/
27642781
if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
27652782
enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
@@ -2779,15 +2796,17 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
27792796
int input_len;
27802797
bool saw_non_ascii = false;
27812798

2782-
/* Make sure space remains in fieldvals[] */
2783-
if (fieldno >= maxfields)
2784-
ereport(ERROR,
2785-
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2786-
errmsg("extra data after last expected column")));
2799+
/* Make sure there is enough space for the next value */
2800+
if (fieldno >= cstate->max_fields)
2801+
{
2802+
cstate->max_fields *= 2;
2803+
cstate->raw_fields =
2804+
repalloc(cstate->raw_fields, cstate->max_fields*sizeof(char *));
2805+
}
27872806

27882807
/* Remember start of field on both input and output sides */
27892808
start_ptr = cur_ptr;
2790-
fieldvals[fieldno] = output_ptr;
2809+
cstate->raw_fields[fieldno] = output_ptr;
27912810

27922811
/* Scan data for field */
27932812
for (;;)
@@ -2912,7 +2931,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
29122931
*/
29132932
if (saw_non_ascii)
29142933
{
2915-
char *fld = fieldvals[fieldno];
2934+
char *fld = cstate->raw_fields[fieldno];
29162935

29172936
pg_verifymbstr(fld, output_ptr - (fld + 1), false);
29182937
}
@@ -2921,7 +2940,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
29212940
input_len = end_ptr - start_ptr;
29222941
if (input_len == cstate->null_print_len &&
29232942
strncmp(start_ptr, cstate->null_print, input_len) == 0)
2924-
fieldvals[fieldno] = NULL;
2943+
cstate->raw_fields[fieldno] = NULL;
29252944

29262945
fieldno++;
29272946
/* Done if we hit EOL instead of a delim */
@@ -2944,7 +2963,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
29442963
* "standard" (i.e. common) CSV usage.
29452964
*/
29462965
static int
2947-
CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
2966+
CopyReadAttributesCSV(CopyState cstate)
29482967
{
29492968
char delimc = cstate->delim[0];
29502969
char quotec = cstate->quote[0];
@@ -2958,7 +2977,7 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
29582977
* We need a special case for zero-column tables: check that the input
29592978
* line is empty, and return.
29602979
*/
2961-
if (maxfields <= 0)
2980+
if (cstate->max_fields <= 0)
29622981
{
29632982
if (cstate->line_buf.len != 0)
29642983
ereport(ERROR,
@@ -2974,7 +2993,7 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
29742993
* data line, so we can just force attribute_buf to be large enough and
29752994
* then transfer data without any checks for enough space. We need to do
29762995
* it this way because enlarging attribute_buf mid-stream would invalidate
2977-
* pointers already stored into fieldvals[].
2996+
* pointers already stored into cstate->raw_fields[].
29782997
*/
29792998
if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
29802999
enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
@@ -2994,15 +3013,17 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
29943013
char *end_ptr;
29953014
int input_len;
29963015

2997-
/* Make sure space remains in fieldvals[] */
2998-
if (fieldno >= maxfields)
2999-
ereport(ERROR,
3000-
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3001-
errmsg("extra data after last expected column")));
3016+
/* Make sure there is enough space for the next value */
3017+
if (fieldno >= cstate->max_fields)
3018+
{
3019+
cstate->max_fields *= 2;
3020+
cstate->raw_fields =
3021+
repalloc(cstate->raw_fields, cstate->max_fields*sizeof(char *));
3022+
}
30023023

30033024
/* Remember start of field on both input and output sides */
30043025
start_ptr = cur_ptr;
3005-
fieldvals[fieldno] = output_ptr;
3026+
cstate->raw_fields[fieldno] = output_ptr;
30063027

30073028
/*
30083029
* Scan data for field,
@@ -3090,7 +3111,7 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
30903111
input_len = end_ptr - start_ptr;
30913112
if (!saw_quote && input_len == cstate->null_print_len &&
30923113
strncmp(start_ptr, cstate->null_print, input_len) == 0)
3093-
fieldvals[fieldno] = NULL;
3114+
cstate->raw_fields[fieldno] = NULL;
30943115

30953116
fieldno++;
30963117
/* Done if we hit EOL instead of a delim */

0 commit comments

Comments
 (0)