Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * copyfromparse.c
4 : : * Parse CSV/text/binary format for COPY FROM.
5 : : *
6 : : * This file contains routines to parse the text, CSV and binary input
7 : : * formats. The main entry point is NextCopyFrom(), which parses the
8 : : * next input line and returns it as Datums.
9 : : *
10 : : * In text/CSV mode, the parsing happens in multiple stages:
11 : : *
12 : : * [data source] --> raw_buf --> input_buf --> line_buf --> attribute_buf
13 : : * 1. 2. 3. 4.
14 : : *
15 : : * 1. CopyLoadRawBuf() reads raw data from the input file or client, and
16 : : * places it into 'raw_buf'.
17 : : *
18 : : * 2. CopyConvertBuf() calls the encoding conversion function to convert
19 : : * the data in 'raw_buf' from client to server encoding, placing the
20 : : * converted result in 'input_buf'.
21 : : *
22 : : * 3. CopyReadLine() parses the data in 'input_buf', one line at a time.
23 : : * It is responsible for finding the next newline marker, taking quote and
24 : : * escape characters into account according to the COPY options. The line
25 : : * is copied into 'line_buf', with quotes and escape characters still
26 : : * intact.
27 : : *
28 : : * 4. CopyReadAttributesText/CSV() function takes the input line from
29 : : * 'line_buf', and splits it into fields, unescaping the data as required.
30 : : * The fields are stored in 'attribute_buf', and 'raw_fields' array holds
31 : : * pointers to each field.
32 : : *
33 : : * If encoding conversion is not required, a shortcut is taken in step 2 to
34 : : * avoid copying the data unnecessarily. The 'input_buf' pointer is set to
35 : : * point directly to 'raw_buf', so that CopyLoadRawBuf() loads the raw data
36 : : * directly into 'input_buf'. CopyConvertBuf() then merely validates that
37 : : * the data is valid in the current encoding.
38 : : *
39 : : * In binary mode, the pipeline is much simpler. Input is loaded into
40 : : * 'raw_buf', and encoding conversion is done in the datatype-specific
41 : : * receive functions, if required. 'input_buf' and 'line_buf' are not used,
42 : : * but 'attribute_buf' is used as a temporary buffer to hold one attribute's
43 : : * data when it's passed the receive function.
44 : : *
45 : : * 'raw_buf' is always 64 kB in size (RAW_BUF_SIZE). 'input_buf' is also
46 : : * 64 kB (INPUT_BUF_SIZE), if encoding conversion is required. 'line_buf'
47 : : * and 'attribute_buf' are expanded on demand, to hold the longest line
48 : : * encountered so far.
49 : : *
50 : : * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
51 : : * Portions Copyright (c) 1994, Regents of the University of California
52 : : *
53 : : *
54 : : * IDENTIFICATION
55 : : * src/backend/commands/copyfromparse.c
56 : : *
57 : : *-------------------------------------------------------------------------
58 : : */
59 : : #include "postgres.h"
60 : :
61 : : #include <ctype.h>
62 : : #include <unistd.h>
63 : : #include <sys/stat.h>
64 : :
65 : : #include "commands/copy.h"
66 : : #include "commands/copyfrom_internal.h"
67 : : #include "commands/progress.h"
68 : : #include "executor/executor.h"
69 : : #include "libpq/libpq.h"
70 : : #include "libpq/pqformat.h"
71 : : #include "mb/pg_wchar.h"
72 : : #include "miscadmin.h"
73 : : #include "nodes/miscnodes.h"
74 : : #include "pgstat.h"
75 : : #include "port/pg_bswap.h"
76 : : #include "utils/builtins.h"
77 : : #include "utils/rel.h"
78 : :
79 : : #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
80 : : #define OCTVALUE(c) ((c) - '0')
81 : :
82 : : /*
83 : : * These macros centralize code used to process line_buf and input_buf buffers.
84 : : * They are macros because they often do continue/break control and to avoid
85 : : * function call overhead in tight COPY loops.
86 : : *
87 : : * We must use "if (1)" because the usual "do {...} while(0)" wrapper would
88 : : * prevent the continue/break processing from working. We end the "if (1)"
89 : : * with "else ((void) 0)" to ensure the "if" does not unintentionally match
90 : : * any "else" in the calling code, and to avoid any compiler warnings about
91 : : * empty statements. See http://www.cit.gu.edu.au/~anthony/info/C/C.macros.
92 : : */
93 : :
94 : : /*
95 : : * This keeps the character read at the top of the loop in the buffer
96 : : * even if there is more than one read-ahead.
97 : : */
98 : : #define IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(extralen) \
99 : : if (1) \
100 : : { \
101 : : if (input_buf_ptr + (extralen) >= copy_buf_len && !hit_eof) \
102 : : { \
103 : : input_buf_ptr = prev_raw_ptr; /* undo fetch */ \
104 : : need_data = true; \
105 : : continue; \
106 : : } \
107 : : } else ((void) 0)
108 : :
109 : : /* This consumes the remainder of the buffer and breaks */
110 : : #define IF_NEED_REFILL_AND_EOF_BREAK(extralen) \
111 : : if (1) \
112 : : { \
113 : : if (input_buf_ptr + (extralen) >= copy_buf_len && hit_eof) \
114 : : { \
115 : : if (extralen) \
116 : : input_buf_ptr = copy_buf_len; /* consume the partial character */ \
117 : : /* backslash just before EOF, treat as data char */ \
118 : : result = true; \
119 : : break; \
120 : : } \
121 : : } else ((void) 0)
122 : :
123 : : /*
124 : : * Transfer any approved data to line_buf; must do this to be sure
125 : : * there is some room in input_buf.
126 : : */
127 : : #define REFILL_LINEBUF \
128 : : if (1) \
129 : : { \
130 : : if (input_buf_ptr > cstate->input_buf_index) \
131 : : { \
132 : : appendBinaryStringInfo(&cstate->line_buf, \
133 : : cstate->input_buf + cstate->input_buf_index, \
134 : : input_buf_ptr - cstate->input_buf_index); \
135 : : cstate->input_buf_index = input_buf_ptr; \
136 : : } \
137 : : } else ((void) 0)
138 : :
139 : : /* Undo any read-ahead and jump out of the block. */
140 : : #define NO_END_OF_COPY_GOTO \
141 : : if (1) \
142 : : { \
143 : : input_buf_ptr = prev_raw_ptr + 1; \
144 : : goto not_end_of_copy; \
145 : : } else ((void) 0)
146 : :
147 : : /* NOTE: there's a copy of this in copyto.c */
148 : : static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
149 : :
150 : :
151 : : /* non-export function prototypes */
152 : : static bool CopyReadLine(CopyFromState cstate);
153 : : static bool CopyReadLineText(CopyFromState cstate);
154 : : static int CopyReadAttributesText(CopyFromState cstate);
155 : : static int CopyReadAttributesCSV(CopyFromState cstate);
156 : : static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
157 : : Oid typioparam, int32 typmod,
158 : : bool *isnull);
159 : :
160 : :
161 : : /* Low-level communications functions */
162 : : static int CopyGetData(CopyFromState cstate, void *databuf,
163 : : int minread, int maxread);
164 : : static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
165 : : static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
166 : : static void CopyLoadInputBuf(CopyFromState cstate);
167 : : static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
168 : :
169 : : void
1238 heikki.linnakangas@i 170 :CBC 446 : ReceiveCopyBegin(CopyFromState cstate)
171 : : {
172 : : StringInfoData buf;
1137 173 : 446 : int natts = list_length(cstate->attnumlist);
174 : 446 : int16 format = (cstate->opts.binary ? 1 : 0);
175 : : int i;
176 : :
236 nathan@postgresql.or 177 :GNC 446 : pq_beginmessage(&buf, PqMsg_CopyInResponse);
1137 heikki.linnakangas@i 178 :CBC 446 : pq_sendbyte(&buf, format); /* overall format */
179 : 446 : pq_sendint16(&buf, natts);
180 [ + + ]: 1502 : for (i = 0; i < natts; i++)
181 : 1056 : pq_sendint16(&buf, format); /* per-column formats */
182 : 446 : pq_endmessage(&buf);
183 : 446 : cstate->copy_src = COPY_FRONTEND;
184 : 446 : cstate->fe_msgbuf = makeStringInfo();
185 : : /* We *must* flush here to ensure FE knows it can send. */
1238 186 : 446 : pq_flush();
187 : 446 : }
188 : :
189 : : void
190 : 7 : ReceiveCopyBinaryHeader(CopyFromState cstate)
191 : : {
192 : : char readSig[11];
193 : : int32 tmp;
194 : :
195 : : /* Signature */
196 [ + - ]: 7 : if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
197 [ - + ]: 7 : memcmp(readSig, BinarySignature, 11) != 0)
1238 heikki.linnakangas@i 198 [ # # ]:UBC 0 : ereport(ERROR,
199 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
200 : : errmsg("COPY file signature not recognized")));
201 : : /* Flags field */
1238 heikki.linnakangas@i 202 [ - + ]:CBC 7 : if (!CopyGetInt32(cstate, &tmp))
1238 heikki.linnakangas@i 203 [ # # ]:UBC 0 : ereport(ERROR,
204 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
205 : : errmsg("invalid COPY file header (missing flags)")));
1238 heikki.linnakangas@i 206 [ - + ]:CBC 7 : if ((tmp & (1 << 16)) != 0)
1238 heikki.linnakangas@i 207 [ # # ]:UBC 0 : ereport(ERROR,
208 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
209 : : errmsg("invalid COPY file header (WITH OIDS)")));
1238 heikki.linnakangas@i 210 :CBC 7 : tmp &= ~(1 << 16);
211 [ - + ]: 7 : if ((tmp >> 16) != 0)
1238 heikki.linnakangas@i 212 [ # # ]:UBC 0 : ereport(ERROR,
213 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
214 : : errmsg("unrecognized critical flags in COPY file header")));
215 : : /* Header extension length */
1238 heikki.linnakangas@i 216 [ + - ]:CBC 7 : if (!CopyGetInt32(cstate, &tmp) ||
217 [ - + ]: 7 : tmp < 0)
1238 heikki.linnakangas@i 218 [ # # ]:UBC 0 : ereport(ERROR,
219 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
220 : : errmsg("invalid COPY file header (missing length)")));
221 : : /* Skip extension header, if present */
1238 heikki.linnakangas@i 222 [ - + ]:CBC 7 : while (tmp-- > 0)
223 : : {
1238 heikki.linnakangas@i 224 [ # # ]:UBC 0 : if (CopyReadBinaryData(cstate, readSig, 1) != 1)
225 [ # # ]: 0 : ereport(ERROR,
226 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
227 : : errmsg("invalid COPY file header (wrong length)")));
228 : : }
1238 heikki.linnakangas@i 229 :CBC 7 : }
230 : :
231 : : /*
232 : : * CopyGetData reads data from the source (file or frontend)
233 : : *
234 : : * We attempt to read at least minread, and at most maxread, bytes from
235 : : * the source. The actual number of bytes read is returned; if this is
236 : : * less than minread, EOF was detected.
237 : : *
238 : : * Note: when copying from the frontend, we expect a proper EOF mark per
239 : : * protocol; if the frontend simply drops the connection, we raise error.
240 : : * It seems unwise to allow the COPY IN to complete normally in that case.
241 : : *
242 : : * NB: no data conversion is applied here.
243 : : */
244 : : static int
245 : 215423 : CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
246 : : {
247 : 215423 : int bytesread = 0;
248 : :
249 [ + + + - ]: 215423 : switch (cstate->copy_src)
250 : : {
251 : 483 : case COPY_FILE:
252 : 483 : bytesread = fread(databuf, 1, maxread, cstate->copy_file);
253 [ - + ]: 483 : if (ferror(cstate->copy_file))
1238 heikki.linnakangas@i 254 [ # # ]:UBC 0 : ereport(ERROR,
255 : : (errcode_for_file_access(),
256 : : errmsg("could not read from COPY file: %m")));
1238 heikki.linnakangas@i 257 [ + + ]:CBC 483 : if (bytesread == 0)
1109 258 : 183 : cstate->raw_reached_eof = true;
1238 259 : 483 : break;
1137 260 : 201002 : case COPY_FRONTEND:
1109 261 [ + - + + : 401607 : while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
+ + ]
262 : : {
263 : : int avail;
264 : :
1238 265 [ + + ]: 401567 : while (cstate->fe_msgbuf->cursor >= cstate->fe_msgbuf->len)
266 : : {
267 : : /* Try to receive another message */
268 : : int mtype;
269 : : int maxmsglen;
270 : :
271 : 200962 : readmessage:
272 : 200962 : HOLD_CANCEL_INTERRUPTS();
273 : 200962 : pq_startmsgread();
274 : 200962 : mtype = pq_getbyte();
275 [ - + ]: 200962 : if (mtype == EOF)
1238 heikki.linnakangas@i 276 [ # # ]:UBC 0 : ereport(ERROR,
277 : : (errcode(ERRCODE_CONNECTION_FAILURE),
278 : : errmsg("unexpected EOF on client connection with an open transaction")));
279 : : /* Validate message type and set packet size limit */
280 : : switch (mtype)
281 : : {
236 nathan@postgresql.or 282 :GNC 200605 : case PqMsg_CopyData:
1082 tgl@sss.pgh.pa.us 283 :CBC 200605 : maxmsglen = PQ_LARGE_MESSAGE_LIMIT;
284 : 200605 : break;
236 nathan@postgresql.or 285 :GNC 357 : case PqMsg_CopyDone:
286 : : case PqMsg_CopyFail:
287 : : case PqMsg_Flush:
288 : : case PqMsg_Sync:
1082 tgl@sss.pgh.pa.us 289 :CBC 357 : maxmsglen = PQ_SMALL_MESSAGE_LIMIT;
290 : 357 : break;
1082 tgl@sss.pgh.pa.us 291 :UBC 0 : default:
292 [ # # ]: 0 : ereport(ERROR,
293 : : (errcode(ERRCODE_PROTOCOL_VIOLATION),
294 : : errmsg("unexpected message type 0x%02X during COPY from stdin",
295 : : mtype)));
296 : : maxmsglen = 0; /* keep compiler quiet */
297 : : break;
298 : : }
299 : : /* Now collect the message body */
1082 tgl@sss.pgh.pa.us 300 [ - + ]:CBC 200962 : if (pq_getmessage(cstate->fe_msgbuf, maxmsglen))
1238 heikki.linnakangas@i 301 [ # # ]:UBC 0 : ereport(ERROR,
302 : : (errcode(ERRCODE_CONNECTION_FAILURE),
303 : : errmsg("unexpected EOF on client connection with an open transaction")));
1238 heikki.linnakangas@i 304 [ - + + + :CBC 200962 : RESUME_CANCEL_INTERRUPTS();
- - - ]
305 : : /* ... and process it */
306 : : switch (mtype)
307 : : {
236 nathan@postgresql.or 308 :GNC 200605 : case PqMsg_CopyData:
1238 heikki.linnakangas@i 309 :CBC 200605 : break;
236 nathan@postgresql.or 310 :GNC 357 : case PqMsg_CopyDone:
311 : : /* COPY IN correctly terminated by frontend */
1109 heikki.linnakangas@i 312 :CBC 357 : cstate->raw_reached_eof = true;
1238 313 : 357 : return bytesread;
236 nathan@postgresql.or 314 :UNC 0 : case PqMsg_CopyFail:
1238 heikki.linnakangas@i 315 [ # # ]:UBC 0 : ereport(ERROR,
316 : : (errcode(ERRCODE_QUERY_CANCELED),
317 : : errmsg("COPY from stdin failed: %s",
318 : : pq_getmsgstring(cstate->fe_msgbuf))));
319 : : break;
236 nathan@postgresql.or 320 :UNC 0 : case PqMsg_Flush:
321 : : case PqMsg_Sync:
322 : :
323 : : /*
324 : : * Ignore Flush/Sync for the convenience of client
325 : : * libraries (such as libpq) that may send those
326 : : * without noticing that the command they just
327 : : * sent was COPY.
328 : : */
1238 heikki.linnakangas@i 329 :UBC 0 : goto readmessage;
330 : 0 : default:
1082 tgl@sss.pgh.pa.us 331 : 0 : Assert(false); /* NOT REACHED */
332 : : }
333 : : }
1238 heikki.linnakangas@i 334 :CBC 200605 : avail = cstate->fe_msgbuf->len - cstate->fe_msgbuf->cursor;
335 [ - + ]: 200605 : if (avail > maxread)
1238 heikki.linnakangas@i 336 :UBC 0 : avail = maxread;
1238 heikki.linnakangas@i 337 :CBC 200605 : pq_copymsgbytes(cstate->fe_msgbuf, databuf, avail);
338 : 200605 : databuf = (void *) ((char *) databuf + avail);
339 : 200605 : maxread -= avail;
340 : 200605 : bytesread += avail;
341 : : }
342 : 200645 : break;
343 : 13938 : case COPY_CALLBACK:
344 : 13938 : bytesread = cstate->data_source_cb(databuf, minread, maxread);
345 : 13938 : break;
346 : : }
347 : :
348 : 215066 : return bytesread;
349 : : }
350 : :
351 : :
352 : : /*
353 : : * These functions do apply some data conversion
354 : : */
355 : :
356 : : /*
357 : : * CopyGetInt32 reads an int32 that appears in network byte order
358 : : *
359 : : * Returns true if OK, false if EOF
360 : : */
361 : : static inline bool
362 : 93 : CopyGetInt32(CopyFromState cstate, int32 *val)
363 : : {
364 : : uint32 buf;
365 : :
366 [ - + ]: 93 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
367 : : {
1238 heikki.linnakangas@i 368 :UBC 0 : *val = 0; /* suppress compiler warning */
369 : 0 : return false;
370 : : }
1238 heikki.linnakangas@i 371 :CBC 93 : *val = (int32) pg_ntoh32(buf);
372 : 93 : return true;
373 : : }
374 : :
375 : : /*
376 : : * CopyGetInt16 reads an int16 that appears in network byte order
377 : : */
378 : : static inline bool
379 : 21 : CopyGetInt16(CopyFromState cstate, int16 *val)
380 : : {
381 : : uint16 buf;
382 : :
383 [ - + ]: 21 : if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
384 : : {
1238 heikki.linnakangas@i 385 :UBC 0 : *val = 0; /* suppress compiler warning */
386 : 0 : return false;
387 : : }
1238 heikki.linnakangas@i 388 :CBC 21 : *val = (int16) pg_ntoh16(buf);
389 : 21 : return true;
390 : : }
391 : :
392 : :
393 : : /*
394 : : * Perform encoding conversion on data in 'raw_buf', writing the converted
395 : : * data into 'input_buf'.
396 : : *
397 : : * On entry, there must be some data to convert in 'raw_buf'.
398 : : */
399 : : static void
1109 400 : 430096 : CopyConvertBuf(CopyFromState cstate)
401 : : {
402 : : /*
403 : : * If the file and server encoding are the same, no encoding conversion is
404 : : * required. However, we still need to verify that the input is valid for
405 : : * the encoding.
406 : : */
407 [ + - ]: 430096 : if (!cstate->need_transcoding)
408 : : {
409 : : /*
410 : : * When conversion is not required, input_buf and raw_buf are the
411 : : * same. raw_buf_len is the total number of bytes in the buffer, and
412 : : * input_buf_len tracks how many of those bytes have already been
413 : : * verified.
414 : : */
415 : 430096 : int preverifiedlen = cstate->input_buf_len;
416 : 430096 : int unverifiedlen = cstate->raw_buf_len - cstate->input_buf_len;
417 : : int nverified;
418 : :
419 [ + + ]: 430096 : if (unverifiedlen == 0)
420 : : {
421 : : /*
422 : : * If no more raw data is coming, report the EOF to the caller.
423 : : */
424 [ + + ]: 215431 : if (cstate->raw_reached_eof)
425 : 383 : cstate->input_reached_eof = true;
426 : 215431 : return;
427 : : }
428 : :
429 : : /*
430 : : * Verify the new data, including any residual unverified bytes from
431 : : * previous round.
432 : : */
433 : 214665 : nverified = pg_encoding_verifymbstr(cstate->file_encoding,
434 : 214665 : cstate->raw_buf + preverifiedlen,
435 : : unverifiedlen);
436 [ - + ]: 214665 : if (nverified == 0)
437 : : {
438 : : /*
439 : : * Could not verify anything.
440 : : *
441 : : * If there is no more raw input data coming, it means that there
442 : : * was an incomplete multi-byte sequence at the end. Also, if
443 : : * there's "enough" input left, we should be able to verify at
444 : : * least one character, and a failure to do so means that we've
445 : : * hit an invalid byte sequence.
446 : : */
686 heikki.linnakangas@i 447 [ # # # # ]:UBC 0 : if (cstate->raw_reached_eof || unverifiedlen >= pg_encoding_max_length(cstate->file_encoding))
1109 448 : 0 : cstate->input_reached_error = true;
449 : 0 : return;
450 : : }
1109 heikki.linnakangas@i 451 :CBC 214665 : cstate->input_buf_len += nverified;
452 : : }
453 : : else
454 : : {
455 : : /*
456 : : * Encoding conversion is needed.
457 : : */
458 : : int nbytes;
459 : : unsigned char *src;
460 : : int srclen;
461 : : unsigned char *dst;
462 : : int dstlen;
463 : : int convertedlen;
464 : :
1109 heikki.linnakangas@i 465 [ # # ]:UBC 0 : if (RAW_BUF_BYTES(cstate) == 0)
466 : : {
467 : : /*
468 : : * If no more raw data is coming, report the EOF to the caller.
469 : : */
470 [ # # ]: 0 : if (cstate->raw_reached_eof)
471 : 0 : cstate->input_reached_eof = true;
472 : 0 : return;
473 : : }
474 : :
475 : : /*
476 : : * First, copy down any unprocessed data.
477 : : */
478 : 0 : nbytes = INPUT_BUF_BYTES(cstate);
479 [ # # # # ]: 0 : if (nbytes > 0 && cstate->input_buf_index > 0)
480 : 0 : memmove(cstate->input_buf, cstate->input_buf + cstate->input_buf_index,
481 : : nbytes);
482 : 0 : cstate->input_buf_index = 0;
483 : 0 : cstate->input_buf_len = nbytes;
484 : 0 : cstate->input_buf[nbytes] = '\0';
485 : :
486 : 0 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
487 : 0 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
488 : 0 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
489 : 0 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
490 : :
491 : : /*
492 : : * Do the conversion. This might stop short, if there is an invalid
493 : : * byte sequence in the input. We'll convert as much as we can in
494 : : * that case.
495 : : *
496 : : * Note: Even if we hit an invalid byte sequence, we don't report the
497 : : * error until all the valid bytes have been consumed. The input
498 : : * might contain an end-of-input marker (\.), and we don't want to
499 : : * report an error if the invalid byte sequence is after the
500 : : * end-of-input marker. We might unnecessarily convert some data
501 : : * after the end-of-input marker as long as it's valid for the
502 : : * encoding, but that's harmless.
503 : : */
504 : 0 : convertedlen = pg_do_encoding_conversion_buf(cstate->conversion_proc,
505 : : cstate->file_encoding,
506 : : GetDatabaseEncoding(),
507 : : src, srclen,
508 : : dst, dstlen,
509 : : true);
510 [ # # ]: 0 : if (convertedlen == 0)
511 : : {
512 : : /*
513 : : * Could not convert anything. If there is no more raw input data
514 : : * coming, it means that there was an incomplete multi-byte
515 : : * sequence at the end. Also, if there is plenty of input left,
516 : : * we should be able to convert at least one character, so a
517 : : * failure to do so must mean that we've hit a byte sequence
518 : : * that's invalid.
519 : : */
520 [ # # # # ]: 0 : if (cstate->raw_reached_eof || srclen >= MAX_CONVERSION_INPUT_LENGTH)
521 : 0 : cstate->input_reached_error = true;
522 : 0 : return;
523 : : }
524 : 0 : cstate->raw_buf_index += convertedlen;
525 : 0 : cstate->input_buf_len += strlen((char *) dst);
526 : : }
527 : : }
528 : :
529 : : /*
530 : : * Report an encoding or conversion error.
531 : : */
532 : : static void
533 : 0 : CopyConversionError(CopyFromState cstate)
534 : : {
535 [ # # ]: 0 : Assert(cstate->raw_buf_len > 0);
536 [ # # ]: 0 : Assert(cstate->input_reached_error);
537 : :
538 [ # # ]: 0 : if (!cstate->need_transcoding)
539 : : {
540 : : /*
541 : : * Everything up to input_buf_len was successfully verified, and
542 : : * input_buf_len points to the invalid or incomplete character.
543 : : */
544 : 0 : report_invalid_encoding(cstate->file_encoding,
545 : 0 : cstate->raw_buf + cstate->input_buf_len,
546 : 0 : cstate->raw_buf_len - cstate->input_buf_len);
547 : : }
548 : : else
549 : : {
550 : : /*
551 : : * raw_buf_index points to the invalid or untranslatable character. We
552 : : * let the conversion routine report the error, because it can provide
553 : : * a more specific error message than we could here. An earlier call
554 : : * to the conversion routine in CopyConvertBuf() detected that there
555 : : * is an error, now we call the conversion routine again with
556 : : * noError=false, to have it throw the error.
557 : : */
558 : : unsigned char *src;
559 : : int srclen;
560 : : unsigned char *dst;
561 : : int dstlen;
562 : :
563 : 0 : src = (unsigned char *) cstate->raw_buf + cstate->raw_buf_index;
564 : 0 : srclen = cstate->raw_buf_len - cstate->raw_buf_index;
565 : 0 : dst = (unsigned char *) cstate->input_buf + cstate->input_buf_len;
566 : 0 : dstlen = INPUT_BUF_SIZE - cstate->input_buf_len + 1;
567 : :
568 : 0 : (void) pg_do_encoding_conversion_buf(cstate->conversion_proc,
569 : : cstate->file_encoding,
570 : : GetDatabaseEncoding(),
571 : : src, srclen,
572 : : dst, dstlen,
573 : : false);
574 : :
575 : : /*
576 : : * The conversion routine should have reported an error, so this
577 : : * should not be reached.
578 : : */
579 [ # # ]: 0 : elog(ERROR, "encoding conversion failed without error");
580 : : }
581 : : }
582 : :
583 : : /*
584 : : * Load more data from data source to raw_buf.
585 : : *
586 : : * If RAW_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the
587 : : * beginning of the buffer, and we load new data after that.
588 : : */
589 : : static void
1238 heikki.linnakangas@i 590 :CBC 215066 : CopyLoadRawBuf(CopyFromState cstate)
591 : : {
592 : : int nbytes;
593 : : int inbytes;
594 : :
595 : : /*
596 : : * In text mode, if encoding conversion is not required, raw_buf and
597 : : * input_buf point to the same buffer. Their len/index better agree, too.
598 : : */
1109 599 [ + + ]: 215066 : if (cstate->raw_buf == cstate->input_buf)
600 : : {
601 [ - + ]: 215048 : Assert(!cstate->need_transcoding);
602 [ - + ]: 215048 : Assert(cstate->raw_buf_index == cstate->input_buf_index);
603 [ - + ]: 215048 : Assert(cstate->input_buf_len <= cstate->raw_buf_len);
604 : : }
605 : :
606 : : /*
607 : : * Copy down the unprocessed data if any.
608 : : */
609 : 215066 : nbytes = RAW_BUF_BYTES(cstate);
610 [ - + - - ]: 215066 : if (nbytes > 0 && cstate->raw_buf_index > 0)
1238 heikki.linnakangas@i 611 :UBC 0 : memmove(cstate->raw_buf, cstate->raw_buf + cstate->raw_buf_index,
612 : : nbytes);
1109 heikki.linnakangas@i 613 :CBC 215066 : cstate->raw_buf_len -= cstate->raw_buf_index;
614 : 215066 : cstate->raw_buf_index = 0;
615 : :
616 : : /*
617 : : * If raw_buf and input_buf are in fact the same buffer, adjust the
618 : : * input_buf variables, too.
619 : : */
620 [ + + ]: 215066 : if (cstate->raw_buf == cstate->input_buf)
621 : : {
622 : 215048 : cstate->input_buf_len -= cstate->input_buf_index;
623 : 215048 : cstate->input_buf_index = 0;
624 : : }
625 : :
626 : : /* Load more data */
627 : 215066 : inbytes = CopyGetData(cstate, cstate->raw_buf + cstate->raw_buf_len,
628 : 215066 : 1, RAW_BUF_SIZE - cstate->raw_buf_len);
1238 629 : 215066 : nbytes += inbytes;
630 : 215066 : cstate->raw_buf[nbytes] = '\0';
631 : 215066 : cstate->raw_buf_len = nbytes;
632 : :
1165 633 : 215066 : cstate->bytes_processed += inbytes;
1194 tomas.vondra@postgre 634 : 215066 : pgstat_progress_update_param(PROGRESS_COPY_BYTES_PROCESSED, cstate->bytes_processed);
635 : :
1109 heikki.linnakangas@i 636 [ + + ]: 215066 : if (inbytes == 0)
637 : 389 : cstate->raw_reached_eof = true;
638 : 215066 : }
639 : :
640 : : /*
641 : : * CopyLoadInputBuf loads some more data into input_buf
642 : : *
643 : : * On return, at least one more input character is loaded into
644 : : * input_buf, or input_reached_eof is set.
645 : : *
646 : : * If INPUT_BUF_BYTES(cstate) > 0, the unprocessed bytes are moved to the start
647 : : * of the buffer and then we load more data after that.
648 : : */
649 : : static void
650 : 215048 : CopyLoadInputBuf(CopyFromState cstate)
651 : : {
652 : 215048 : int nbytes = INPUT_BUF_BYTES(cstate);
653 : :
654 : : /*
655 : : * The caller has updated input_buf_index to indicate how much of the
656 : : * input has been consumed and isn't needed anymore. If input_buf is the
657 : : * same physical area as raw_buf, update raw_buf_index accordingly.
658 : : */
659 [ + - ]: 215048 : if (cstate->raw_buf == cstate->input_buf)
660 : : {
661 [ - + ]: 215048 : Assert(!cstate->need_transcoding);
662 [ - + ]: 215048 : Assert(cstate->input_buf_index >= cstate->raw_buf_index);
663 : 215048 : cstate->raw_buf_index = cstate->input_buf_index;
664 : : }
665 : :
666 : : for (;;)
667 : : {
668 : : /* If we now have some unconverted data, try to convert it */
669 : 430096 : CopyConvertBuf(cstate);
670 : :
671 : : /* If we now have some more input bytes ready, return them */
672 [ + + ]: 430096 : if (INPUT_BUF_BYTES(cstate) > nbytes)
673 : 214665 : return;
674 : :
675 : : /*
676 : : * If we reached an invalid byte sequence, or we're at an incomplete
677 : : * multi-byte character but there is no more raw input data, report
678 : : * conversion error.
679 : : */
680 [ - + ]: 215431 : if (cstate->input_reached_error)
1109 heikki.linnakangas@i 681 :UBC 0 : CopyConversionError(cstate);
682 : :
683 : : /* no more input, and everything has been converted */
1109 heikki.linnakangas@i 684 [ + + ]:CBC 215431 : if (cstate->input_reached_eof)
685 : 383 : break;
686 : :
687 : : /* Try to load more raw data */
688 [ - + ]: 215048 : Assert(!cstate->raw_reached_eof);
689 : 215048 : CopyLoadRawBuf(cstate);
690 : : }
691 : : }
692 : :
693 : : /*
694 : : * CopyReadBinaryData
695 : : *
696 : : * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
697 : : * and writes them to 'dest'. Returns the number of bytes read (which
698 : : * would be less than 'nbytes' only if we reach EOF).
699 : : */
700 : : static int
1238 701 : 191 : CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
702 : : {
703 : 191 : int copied_bytes = 0;
704 : :
705 [ + + ]: 191 : if (RAW_BUF_BYTES(cstate) >= nbytes)
706 : : {
707 : : /* Enough bytes are present in the buffer. */
708 : 173 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, nbytes);
709 : 173 : cstate->raw_buf_index += nbytes;
710 : 173 : copied_bytes = nbytes;
711 : : }
712 : : else
713 : : {
714 : : /*
715 : : * Not enough bytes in the buffer, so must read from the file. Need
716 : : * to loop since 'nbytes' could be larger than the buffer size.
717 : : */
718 : : do
719 : : {
720 : : int copy_bytes;
721 : :
722 : : /* Load more data if buffer is empty. */
723 [ + - ]: 18 : if (RAW_BUF_BYTES(cstate) == 0)
724 : : {
1109 725 : 18 : CopyLoadRawBuf(cstate);
726 [ + + ]: 18 : if (cstate->raw_reached_eof)
1238 727 : 6 : break; /* EOF */
728 : : }
729 : :
730 : : /* Transfer some bytes. */
731 : 12 : copy_bytes = Min(nbytes - copied_bytes, RAW_BUF_BYTES(cstate));
732 : 12 : memcpy(dest, cstate->raw_buf + cstate->raw_buf_index, copy_bytes);
733 : 12 : cstate->raw_buf_index += copy_bytes;
734 : 12 : dest += copy_bytes;
735 : 12 : copied_bytes += copy_bytes;
736 [ - + ]: 12 : } while (copied_bytes < nbytes);
737 : : }
738 : :
739 : 191 : return copied_bytes;
740 : : }
741 : :
742 : : /*
743 : : * Read raw fields in the next line for COPY FROM in text or csv mode.
744 : : * Return false if no more lines.
745 : : *
746 : : * An internal temporary buffer is returned via 'fields'. It is valid until
747 : : * the next call of the function. Since the function returns all raw fields
748 : : * in the input file, 'nfields' could be different from the number of columns
749 : : * in the relation.
750 : : *
751 : : * NOTE: force_not_null option are not applied to the returned fields.
752 : : */
753 : : bool
754 : 724291 : NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
755 : : {
756 : : int fldct;
757 : : bool done;
758 : :
759 : : /* only available for text or csv input */
760 [ - + ]: 724291 : Assert(!cstate->opts.binary);
761 : :
762 : : /* on input check that the header line is correct if needed */
763 [ + + + + ]: 724291 : if (cstate->cur_lineno == 0 && cstate->opts.header_line)
764 : : {
765 : : ListCell *cur;
766 : : TupleDesc tupDesc;
767 : :
746 peter@eisentraut.org 768 : 55 : tupDesc = RelationGetDescr(cstate->rel);
769 : :
1238 heikki.linnakangas@i 770 : 55 : cstate->cur_lineno++;
746 peter@eisentraut.org 771 : 55 : done = CopyReadLine(cstate);
772 : :
773 [ + + ]: 55 : if (cstate->opts.header_line == COPY_HEADER_MATCH)
774 : : {
775 : : int fldnum;
776 : :
60 michael@paquier.xyz 777 [ + + ]: 38 : if (cstate->opts.csv_mode)
778 : 5 : fldct = CopyReadAttributesCSV(cstate);
779 : : else
780 : 33 : fldct = CopyReadAttributesText(cstate);
781 : :
746 peter@eisentraut.org 782 [ + + ]: 38 : if (fldct != list_length(cstate->attnumlist))
783 [ + - ]: 12 : ereport(ERROR,
784 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
785 : : errmsg("wrong number of fields in header line: got %d, expected %d",
786 : : fldct, list_length(cstate->attnumlist))));
787 : :
788 : 26 : fldnum = 0;
789 [ + - + + : 79 : foreach(cur, cstate->attnumlist)
+ + ]
790 : : {
791 : 63 : int attnum = lfirst_int(cur);
792 : : char *colName;
793 : 63 : Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
794 : :
661 michael@paquier.xyz 795 [ - + ]: 63 : Assert(fldnum < cstate->max_fields);
796 : :
797 : 63 : colName = cstate->raw_fields[fldnum++];
746 peter@eisentraut.org 798 [ + + ]: 63 : if (colName == NULL)
799 [ + - ]: 3 : ereport(ERROR,
800 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
801 : : errmsg("column name mismatch in header line field %d: got null value (\"%s\"), expected \"%s\"",
802 : : fldnum, cstate->opts.null_print, NameStr(attr->attname))));
803 : :
703 tgl@sss.pgh.pa.us 804 [ + + ]: 60 : if (namestrcmp(&attr->attname, colName) != 0)
805 : : {
746 peter@eisentraut.org 806 [ + - ]: 7 : ereport(ERROR,
807 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
808 : : errmsg("column name mismatch in header line field %d: got \"%s\", expected \"%s\"",
809 : : fldnum, colName, NameStr(attr->attname))));
810 : : }
811 : : }
812 : : }
813 : :
814 [ - + ]: 33 : if (done)
746 peter@eisentraut.org 815 :UBC 0 : return false;
816 : : }
817 : :
1238 heikki.linnakangas@i 818 :CBC 724269 : cstate->cur_lineno++;
819 : :
820 : : /* Actually read the line into memory here */
821 : 724269 : done = CopyReadLine(cstate);
822 : :
823 : : /*
824 : : * EOF at start of line means we're done. If we see EOF after some
825 : : * characters, we act as though it was newline followed by EOF, ie,
826 : : * process the line and then exit loop on next iteration.
827 : : */
828 [ + + + - ]: 724269 : if (done && cstate->line_buf.len == 0)
829 : 700 : return false;
830 : :
831 : : /* Parse the line into de-escaped field values */
60 michael@paquier.xyz 832 [ + + ]: 723569 : if (cstate->opts.csv_mode)
833 : 197 : fldct = CopyReadAttributesCSV(cstate);
834 : : else
835 : 723372 : fldct = CopyReadAttributesText(cstate);
836 : :
1238 heikki.linnakangas@i 837 : 723563 : *fields = cstate->raw_fields;
838 : 723563 : *nfields = fldct;
839 : 723563 : return true;
840 : : }
841 : :
842 : : /*
843 : : * Read next tuple from file for COPY FROM. Return false if no more tuples.
844 : : *
845 : : * 'econtext' is used to evaluate default expression for each column that is
846 : : * either not read from the file or is using the DEFAULT option of COPY FROM.
847 : : * It can be NULL when no default values are used, i.e. when all columns are
848 : : * read from the file, and DEFAULT option is unset.
849 : : *
850 : : * 'values' and 'nulls' arrays must be the same length as columns of the
851 : : * relation passed to BeginCopyFrom. This function fills the arrays.
852 : : */
853 : : bool
854 : 724312 : NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
855 : : Datum *values, bool *nulls)
856 : : {
857 : : TupleDesc tupDesc;
858 : : AttrNumber num_phys_attrs,
859 : : attr_count,
860 : 724312 : num_defaults = cstate->num_defaults;
861 : 724312 : FmgrInfo *in_functions = cstate->in_functions;
862 : 724312 : Oid *typioparams = cstate->typioparams;
863 : : int i;
864 : 724312 : int *defmap = cstate->defmap;
865 : 724312 : ExprState **defexprs = cstate->defexprs;
866 : :
867 : 724312 : tupDesc = RelationGetDescr(cstate->rel);
868 : 724312 : num_phys_attrs = tupDesc->natts;
869 : 724312 : attr_count = list_length(cstate->attnumlist);
870 : :
871 : : /* Initialize all values for row to NULL */
872 [ + - + - : 3094680 : MemSet(values, 0, num_phys_attrs * sizeof(Datum));
+ - + - +
+ ]
873 [ + - + + : 724312 : MemSet(nulls, true, num_phys_attrs * sizeof(bool));
- + - - -
- ]
262 drowley@postgresql.o 874 [ + - + + : 796366 : MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+ - + - +
+ ]
875 : :
1238 heikki.linnakangas@i 876 [ + + ]: 724312 : if (!cstate->opts.binary)
877 : : {
878 : : char **field_strings;
879 : : ListCell *cur;
880 : : int fldct;
881 : : int fieldno;
882 : : char *string;
883 : :
884 : : /* read raw fields in the next line */
885 [ + + ]: 724291 : if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
886 : 721 : return false;
887 : :
888 : : /* check for overflowing fields */
889 [ + + + + ]: 723563 : if (attr_count > 0 && fldct > attr_count)
890 [ + - ]: 9 : ereport(ERROR,
891 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
892 : : errmsg("extra data after last expected column")));
893 : :
894 : 723554 : fieldno = 0;
895 : :
896 : : /* Loop to read the user attributes on the line. */
897 [ + + + + : 3033163 : foreach(cur, cstate->attnumlist)
+ + ]
898 : : {
899 : 2309658 : int attnum = lfirst_int(cur);
900 : 2309658 : int m = attnum - 1;
901 : 2309658 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
902 : :
903 [ + + ]: 2309658 : if (fieldno >= fldct)
904 [ + - ]: 9 : ereport(ERROR,
905 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
906 : : errmsg("missing data for column \"%s\"",
907 : : NameStr(att->attname))));
908 : 2309649 : string = field_strings[fieldno++];
909 : :
910 [ + + ]: 2309649 : if (cstate->convert_select_flags &&
911 [ + + ]: 10 : !cstate->convert_select_flags[m])
912 : : {
913 : : /* ignore input field, leaving column as NULL */
914 : 5 : continue;
915 : : }
916 : :
917 [ + + ]: 2309644 : if (cstate->opts.csv_mode)
918 : : {
919 [ + + ]: 426 : if (string == NULL &&
920 [ + + ]: 22 : cstate->opts.force_notnull_flags[m])
921 : : {
922 : : /*
923 : : * FORCE_NOT_NULL option is set and column is NULL -
924 : : * convert it to the NULL string.
925 : : */
926 : 14 : string = cstate->opts.null_print;
927 : : }
928 [ + + + + ]: 412 : else if (string != NULL && cstate->opts.force_null_flags[m]
929 [ + + ]: 25 : && strcmp(string, cstate->opts.null_print) == 0)
930 : : {
931 : : /*
932 : : * FORCE_NULL option is set and column matches the NULL
933 : : * string. It must have been quoted, or otherwise the
934 : : * string would already have been set to NULL. Convert it
935 : : * to NULL as specified.
936 : : */
937 : 13 : string = NULL;
938 : : }
939 : : }
940 : :
941 : 2309644 : cstate->cur_attname = NameStr(att->attname);
942 : 2309644 : cstate->cur_attval = string;
943 : :
944 [ + + ]: 2309644 : if (string != NULL)
945 : 2307225 : nulls[m] = false;
946 : :
398 andrew@dunslane.net 947 [ + + ]: 2309644 : if (cstate->defaults[m])
948 : : {
949 : : /*
950 : : * The caller must supply econtext and have switched into the
951 : : * per-tuple memory context in it.
952 : : */
953 [ - + ]: 30 : Assert(econtext != NULL);
954 [ - + ]: 30 : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
955 : :
956 : 30 : values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
957 : : }
958 : :
959 : : /*
960 : : * If ON_ERROR is specified with IGNORE, skip rows with soft
961 : : * errors
962 : : */
89 akorotkov@postgresql 963 [ + + ]:GNC 2309595 : else if (!InputFunctionCallSafe(&in_functions[m],
964 : : string,
965 : 2309614 : typioparams[m],
966 : : att->atttypmod,
967 : 2309614 : (Node *) cstate->escontext,
968 : 2309614 : &values[m]))
969 : : {
13 msawada@postgresql.o 970 [ - + ]: 21 : Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP);
971 : :
89 akorotkov@postgresql 972 : 21 : cstate->num_errors++;
973 : :
13 msawada@postgresql.o 974 [ + - ]: 21 : if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE)
975 : : {
976 : : /*
977 : : * Since we emit line number and column info in the below
978 : : * notice message, we suppress error context information
979 : : * other than the relation name.
980 : : */
981 [ - + ]: 21 : Assert(!cstate->relname_only);
982 : 21 : cstate->relname_only = true;
983 : :
984 [ + + ]: 21 : if (cstate->cur_attval)
985 : : {
986 : : char *attval;
987 : :
988 : 18 : attval = CopyLimitPrintoutLength(cstate->cur_attval);
989 [ + - ]: 18 : ereport(NOTICE,
990 : : errmsg("skipping row due to data type incompatibility at line %llu for column %s: \"%s\"",
991 : : (unsigned long long) cstate->cur_lineno,
992 : : cstate->cur_attname,
993 : : attval));
994 : 18 : pfree(attval);
995 : : }
996 : : else
997 [ + - ]: 3 : ereport(NOTICE,
998 : : errmsg("skipping row due to data type incompatibility at line %llu for column %s: null input",
999 : : (unsigned long long) cstate->cur_lineno,
1000 : : cstate->cur_attname));
1001 : :
1002 : : /* reset relname_only */
1003 : 21 : cstate->relname_only = false;
1004 : : }
1005 : :
89 akorotkov@postgresql 1006 : 21 : return true;
1007 : : }
1008 : :
1238 heikki.linnakangas@i 1009 :CBC 2309604 : cstate->cur_attname = NULL;
1010 : 2309604 : cstate->cur_attval = NULL;
1011 : : }
1012 : :
1013 [ - + ]: 723505 : Assert(fieldno == attr_count);
1014 : : }
1015 : : else
1016 : : {
1017 : : /* binary */
1018 : : int16 fld_count;
1019 : : ListCell *cur;
1020 : :
1021 : 21 : cstate->cur_lineno++;
1022 : :
1023 [ - + ]: 21 : if (!CopyGetInt16(cstate, &fld_count))
1024 : : {
1025 : : /* EOF detected (end of file, or protocol-level EOF) */
1238 heikki.linnakangas@i 1026 :UBC 0 : return false;
1027 : : }
1028 : :
1238 heikki.linnakangas@i 1029 [ + + ]:CBC 21 : if (fld_count == -1)
1030 : : {
1031 : : /*
1032 : : * Received EOF marker. Wait for the protocol-level EOF, and
1033 : : * complain if it doesn't come immediately. In COPY FROM STDIN,
1034 : : * this ensures that we correctly handle CopyFail, if client
1035 : : * chooses to send that now. When copying from file, we could
1036 : : * ignore the rest of the file like in text mode, but we choose to
1037 : : * be consistent with the COPY FROM STDIN case.
1038 : : */
1039 : : char dummy;
1040 : :
1137 1041 [ - + ]: 6 : if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
1238 heikki.linnakangas@i 1042 [ # # ]:UBC 0 : ereport(ERROR,
1043 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1044 : : errmsg("received copy data after EOF marker")));
1238 heikki.linnakangas@i 1045 :CBC 6 : return false;
1046 : : }
1047 : :
1048 [ - + ]: 15 : if (fld_count != attr_count)
1238 heikki.linnakangas@i 1049 [ # # ]:UBC 0 : ereport(ERROR,
1050 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1051 : : errmsg("row field count is %d, expected %d",
1052 : : (int) fld_count, attr_count)));
1053 : :
1238 heikki.linnakangas@i 1054 [ + - + + :CBC 93 : foreach(cur, cstate->attnumlist)
+ + ]
1055 : : {
1056 : 79 : int attnum = lfirst_int(cur);
1057 : 79 : int m = attnum - 1;
1058 : 79 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1059 : :
1060 : 79 : cstate->cur_attname = NameStr(att->attname);
1061 : 157 : values[m] = CopyReadBinaryAttribute(cstate,
1062 : 79 : &in_functions[m],
1063 : 79 : typioparams[m],
1064 : : att->atttypmod,
1065 : : &nulls[m]);
1066 : 78 : cstate->cur_attname = NULL;
1067 : : }
1068 : : }
1069 : :
1070 : : /*
1071 : : * Now compute and insert any defaults available for the columns not
1072 : : * provided by the input data. Anything not processed here or above will
1073 : : * remain NULL.
1074 : : */
1075 [ + + ]: 753784 : for (i = 0; i < num_defaults; i++)
1076 : : {
1077 : : /*
1078 : : * The caller must supply econtext and have switched into the
1079 : : * per-tuple memory context in it.
1080 : : */
1081 [ - + ]: 30265 : Assert(econtext != NULL);
1082 [ - + ]: 30265 : Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
1083 : :
398 andrew@dunslane.net 1084 : 30265 : values[defmap[i]] = ExecEvalExpr(defexprs[defmap[i]], econtext,
1238 heikki.linnakangas@i 1085 : 30265 : &nulls[defmap[i]]);
1086 : : }
1087 : :
1088 : 723519 : return true;
1089 : : }
1090 : :
1091 : : /*
1092 : : * Read the next input line and stash it in line_buf.
1093 : : *
1094 : : * Result is true if read was terminated by EOF, false if terminated
1095 : : * by newline. The terminating newline or EOF marker is not included
1096 : : * in the final value of line_buf.
1097 : : */
1098 : : static bool
1099 : 724324 : CopyReadLine(CopyFromState cstate)
1100 : : {
1101 : : bool result;
1102 : :
1103 : 724324 : resetStringInfo(&cstate->line_buf);
1109 1104 : 724324 : cstate->line_buf_valid = false;
1105 : :
1106 : : /* Parse data and transfer into line_buf */
1238 1107 : 724324 : result = CopyReadLineText(cstate);
1108 : :
1109 [ + + ]: 724324 : if (result)
1110 : : {
1111 : : /*
1112 : : * Reached EOF. In protocol version 3, we should ignore anything
1113 : : * after \. up to the protocol end of copy data. (XXX maybe better
1114 : : * not to treat \. as special?)
1115 : : */
1137 1116 [ + + ]: 700 : if (cstate->copy_src == COPY_FRONTEND)
1117 : : {
1118 : : int inbytes;
1119 : :
1120 : : do
1121 : : {
1109 1122 : 357 : inbytes = CopyGetData(cstate, cstate->input_buf,
1123 : : 1, INPUT_BUF_SIZE);
1124 [ - + ]: 357 : } while (inbytes > 0);
1125 : 357 : cstate->input_buf_index = 0;
1126 : 357 : cstate->input_buf_len = 0;
1127 : 357 : cstate->raw_buf_index = 0;
1128 : 357 : cstate->raw_buf_len = 0;
1129 : : }
1130 : : }
1131 : : else
1132 : : {
1133 : : /*
1134 : : * If we didn't hit EOF, then we must have transferred the EOL marker
1135 : : * to line_buf along with the data. Get rid of it.
1136 : : */
1238 1137 [ + - - - : 723624 : switch (cstate->eol_type)
- ]
1138 : : {
1139 : 723624 : case EOL_NL:
1140 [ - + ]: 723624 : Assert(cstate->line_buf.len >= 1);
1141 [ - + ]: 723624 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1142 : 723624 : cstate->line_buf.len--;
1143 : 723624 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1144 : 723624 : break;
1238 heikki.linnakangas@i 1145 :UBC 0 : case EOL_CR:
1146 [ # # ]: 0 : Assert(cstate->line_buf.len >= 1);
1147 [ # # ]: 0 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\r');
1148 : 0 : cstate->line_buf.len--;
1149 : 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1150 : 0 : break;
1151 : 0 : case EOL_CRNL:
1152 [ # # ]: 0 : Assert(cstate->line_buf.len >= 2);
1153 [ # # ]: 0 : Assert(cstate->line_buf.data[cstate->line_buf.len - 2] == '\r');
1154 [ # # ]: 0 : Assert(cstate->line_buf.data[cstate->line_buf.len - 1] == '\n');
1155 : 0 : cstate->line_buf.len -= 2;
1156 : 0 : cstate->line_buf.data[cstate->line_buf.len] = '\0';
1157 : 0 : break;
1158 : 0 : case EOL_UNKNOWN:
1159 : : /* shouldn't get here */
1160 : 0 : Assert(false);
1161 : : break;
1162 : : }
1163 : : }
1164 : :
1165 : : /* Now it's safe to use the buffer in error messages */
1109 heikki.linnakangas@i 1166 :CBC 724324 : cstate->line_buf_valid = true;
1167 : :
1238 1168 : 724324 : return result;
1169 : : }
1170 : :
1171 : : /*
1172 : : * CopyReadLineText - inner loop of CopyReadLine for text mode
1173 : : */
1174 : : static bool
1175 : 724324 : CopyReadLineText(CopyFromState cstate)
1176 : : {
1177 : : char *copy_input_buf;
1178 : : int input_buf_ptr;
1179 : : int copy_buf_len;
1180 : 724324 : bool need_data = false;
1181 : 724324 : bool hit_eof = false;
1182 : 724324 : bool result = false;
1183 : :
1184 : : /* CSV variables */
1185 : 724324 : bool first_char_in_line = true;
1186 : 724324 : bool in_quote = false,
1187 : 724324 : last_was_esc = false;
1188 : 724324 : char quotec = '\0';
1189 : 724324 : char escapec = '\0';
1190 : :
1191 [ + + ]: 724324 : if (cstate->opts.csv_mode)
1192 : : {
1193 : 301 : quotec = cstate->opts.quote[0];
1194 : 301 : escapec = cstate->opts.escape[0];
1195 : : /* ignore special escape processing if it's the same as quotec */
1196 [ + + ]: 301 : if (quotec == escapec)
1197 : 233 : escapec = '\0';
1198 : : }
1199 : :
1200 : : /*
1201 : : * The objective of this loop is to transfer the entire next input line
1202 : : * into line_buf. Hence, we only care for detecting newlines (\r and/or
1203 : : * \n) and the end-of-copy marker (\.).
1204 : : *
1205 : : * In CSV mode, \r and \n inside a quoted field are just part of the data
1206 : : * value and are put in line_buf. We keep just enough state to know if we
1207 : : * are currently in a quoted field or not.
1208 : : *
1209 : : * The input has already been converted to the database encoding. All
1210 : : * supported server encodings have the property that all bytes in a
1211 : : * multi-byte sequence have the high bit set, so a multibyte character
1212 : : * cannot contain any newline or escape characters embedded in the
1213 : : * multibyte sequence. Therefore, we can process the input byte-by-byte,
1214 : : * regardless of the encoding.
1215 : : *
1216 : : * For speed, we try to move data from input_buf to line_buf in chunks
1217 : : * rather than one character at a time. input_buf_ptr points to the next
1218 : : * character to examine; any characters from input_buf_index to
1219 : : * input_buf_ptr have been determined to be part of the line, but not yet
1220 : : * transferred to line_buf.
1221 : : *
1222 : : * For a little extra speed within the loop, we copy input_buf and
1223 : : * input_buf_len into local variables.
1224 : : */
1109 1225 : 724324 : copy_input_buf = cstate->input_buf;
1226 : 724324 : input_buf_ptr = cstate->input_buf_index;
1227 : 724324 : copy_buf_len = cstate->input_buf_len;
1228 : :
1229 : : for (;;)
1238 1230 : 12830558 : {
1231 : : int prev_raw_ptr;
1232 : : char c;
1233 : :
1234 : : /*
1235 : : * Load more data if needed.
1236 : : *
1237 : : * TODO: We could just force four bytes of read-ahead and avoid the
1238 : : * many calls to IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(). That was
1239 : : * unsafe with the old v2 COPY protocol, but we don't support that
1240 : : * anymore.
1241 : : */
1109 1242 [ + + - + ]: 13554882 : if (input_buf_ptr >= copy_buf_len || need_data)
1243 : : {
1238 1244 [ + + ]: 215048 : REFILL_LINEBUF;
1245 : :
1109 1246 : 215048 : CopyLoadInputBuf(cstate);
1247 : : /* update our local variables */
1248 : 215048 : hit_eof = cstate->input_reached_eof;
1249 : 215048 : input_buf_ptr = cstate->input_buf_index;
1250 : 215048 : copy_buf_len = cstate->input_buf_len;
1251 : :
1252 : : /*
1253 : : * If we are completely out of data, break out of the loop,
1254 : : * reporting EOF.
1255 : : */
1256 [ + + ]: 215048 : if (INPUT_BUF_BYTES(cstate) <= 0)
1257 : : {
1238 1258 : 383 : result = true;
1259 : 383 : break;
1260 : : }
1261 : 214665 : need_data = false;
1262 : : }
1263 : :
1264 : : /* OK to fetch a character */
1109 1265 : 13554499 : prev_raw_ptr = input_buf_ptr;
1266 : 13554499 : c = copy_input_buf[input_buf_ptr++];
1267 : :
1238 1268 [ + + ]: 13554499 : if (cstate->opts.csv_mode)
1269 : : {
1270 : : /*
1271 : : * If character is '\\' or '\r', we may need to look ahead below.
1272 : : * Force fetch of the next character if we don't already have it.
1273 : : * We need to do this before changing CSV state, in case one of
1274 : : * these characters is also the quote or escape character.
1275 : : */
1276 [ + + + + ]: 2429 : if (c == '\\' || c == '\r')
1277 : : {
1278 [ - + - - ]: 159 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1279 : : }
1280 : :
1281 : : /*
1282 : : * Dealing with quotes and escapes here is mildly tricky. If the
1283 : : * quote char is also the escape char, there's no problem - we
1284 : : * just use the char as a toggle. If they are different, we need
1285 : : * to ensure that we only take account of an escape inside a
1286 : : * quoted field and immediately preceding a quote char, and not
1287 : : * the second in an escape-escape sequence.
1288 : : */
1289 [ + + + + ]: 2429 : if (in_quote && c == escapec)
1290 : 24 : last_was_esc = !last_was_esc;
1291 [ + + + - ]: 2429 : if (c == quotec && !last_was_esc)
1292 : 204 : in_quote = !in_quote;
1293 [ + + ]: 2429 : if (c != escapec)
1294 : 2402 : last_was_esc = false;
1295 : :
1296 : : /*
1297 : : * Updating the line count for embedded CR and/or LF chars is
1298 : : * necessarily a little fragile - this test is probably about the
1299 : : * best we can do. (XXX it's arguable whether we should do this
1300 : : * at all --- is cur_lineno a physical or logical count?)
1301 : : */
1302 [ + + + + : 2429 : if (in_quote && c == (cstate->eol_type == EOL_NL ? '\n' : '\r'))
+ + ]
1303 : 18 : cstate->cur_lineno++;
1304 : : }
1305 : :
1306 : : /* Process \r */
1307 [ + + + - : 13554499 : if (c == '\r' && (!cstate->opts.csv_mode || !in_quote))
- + ]
1308 : : {
1309 : : /* Check for \r\n on first line, _and_ handle \r\n. */
1238 heikki.linnakangas@i 1310 [ # # ]:UBC 0 : if (cstate->eol_type == EOL_UNKNOWN ||
1311 [ # # ]: 0 : cstate->eol_type == EOL_CRNL)
1312 : : {
1313 : : /*
1314 : : * If need more data, go back to loop top to load it.
1315 : : *
1316 : : * Note that if we are at EOF, c will wind up as '\0' because
1317 : : * of the guaranteed pad of input_buf.
1318 : : */
1319 [ # # # # ]: 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1320 : :
1321 : : /* get next char */
1109 1322 : 0 : c = copy_input_buf[input_buf_ptr];
1323 : :
1238 1324 [ # # ]: 0 : if (c == '\n')
1325 : : {
1109 1326 : 0 : input_buf_ptr++; /* eat newline */
1238 1327 : 0 : cstate->eol_type = EOL_CRNL; /* in case not set yet */
1328 : : }
1329 : : else
1330 : : {
1331 : : /* found \r, but no \n */
1332 [ # # ]: 0 : if (cstate->eol_type == EOL_CRNL)
1333 [ # # # # : 0 : ereport(ERROR,
# # ]
1334 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1335 : : !cstate->opts.csv_mode ?
1336 : : errmsg("literal carriage return found in data") :
1337 : : errmsg("unquoted carriage return found in data"),
1338 : : !cstate->opts.csv_mode ?
1339 : : errhint("Use \"\\r\" to represent carriage return.") :
1340 : : errhint("Use quoted CSV field to represent carriage return.")));
1341 : :
1342 : : /*
1343 : : * if we got here, it is the first line and we didn't find
1344 : : * \n, so don't consume the peeked character
1345 : : */
1346 : 0 : cstate->eol_type = EOL_CR;
1347 : : }
1348 : : }
1349 [ # # ]: 0 : else if (cstate->eol_type == EOL_NL)
1350 [ # # # # : 0 : ereport(ERROR,
# # ]
1351 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1352 : : !cstate->opts.csv_mode ?
1353 : : errmsg("literal carriage return found in data") :
1354 : : errmsg("unquoted carriage return found in data"),
1355 : : !cstate->opts.csv_mode ?
1356 : : errhint("Use \"\\r\" to represent carriage return.") :
1357 : : errhint("Use quoted CSV field to represent carriage return.")));
1358 : : /* If reach here, we have found the line terminator */
1359 : 0 : break;
1360 : : }
1361 : :
1362 : : /* Process \n */
1238 heikki.linnakangas@i 1363 [ + + + + :CBC 13554499 : if (c == '\n' && (!cstate->opts.csv_mode || !in_quote))
+ + ]
1364 : : {
1365 [ + - - + ]: 723624 : if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
1238 heikki.linnakangas@i 1366 [ # # # # :UBC 0 : ereport(ERROR,
# # ]
1367 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1368 : : !cstate->opts.csv_mode ?
1369 : : errmsg("literal newline found in data") :
1370 : : errmsg("unquoted newline found in data"),
1371 : : !cstate->opts.csv_mode ?
1372 : : errhint("Use \"\\n\" to represent newline.") :
1373 : : errhint("Use quoted CSV field to represent newline.")));
1238 heikki.linnakangas@i 1374 :CBC 723624 : cstate->eol_type = EOL_NL; /* in case not set yet */
1375 : : /* If reach here, we have found the line terminator */
1376 : 723624 : break;
1377 : : }
1378 : :
1379 : : /*
1380 : : * In CSV mode, we only recognize \. alone on a line. This is because
1381 : : * \. is a valid CSV data value.
1382 : : */
1383 [ + + + + : 12830875 : if (c == '\\' && (!cstate->opts.csv_mode || first_char_in_line))
+ + ]
1384 : : {
1385 : : char c2;
1386 : :
1387 [ - + - - ]: 4317 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1388 [ - + - - ]: 4317 : IF_NEED_REFILL_AND_EOF_BREAK(0);
1389 : :
1390 : : /* -----
1391 : : * get next character
1392 : : * Note: we do not change c so if it isn't \., we can fall
1393 : : * through and continue processing.
1394 : : * -----
1395 : : */
1109 1396 : 4317 : c2 = copy_input_buf[input_buf_ptr];
1397 : :
1238 1398 [ + + ]: 4317 : if (c2 == '.')
1399 : : {
1109 1400 : 320 : input_buf_ptr++; /* consume the '.' */
1401 : :
1402 : : /*
1403 : : * Note: if we loop back for more data here, it does not
1404 : : * matter that the CSV state change checks are re-executed; we
1405 : : * will come back here with no important state changed.
1406 : : */
1238 1407 [ - + ]: 320 : if (cstate->eol_type == EOL_CRNL)
1408 : : {
1409 : : /* Get the next character */
1238 heikki.linnakangas@i 1410 [ # # # # ]:UBC 0 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1411 : : /* if hit_eof, c2 will become '\0' */
1109 1412 : 0 : c2 = copy_input_buf[input_buf_ptr++];
1413 : :
1238 1414 [ # # ]: 0 : if (c2 == '\n')
1415 : : {
1416 [ # # ]: 0 : if (!cstate->opts.csv_mode)
1417 [ # # ]: 0 : ereport(ERROR,
1418 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1419 : : errmsg("end-of-copy marker does not match previous newline style")));
1420 : : else
1421 : 0 : NO_END_OF_COPY_GOTO;
1422 : : }
1423 [ # # ]: 0 : else if (c2 != '\r')
1424 : : {
1425 [ # # ]: 0 : if (!cstate->opts.csv_mode)
1426 [ # # ]: 0 : ereport(ERROR,
1427 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1428 : : errmsg("end-of-copy marker corrupt")));
1429 : : else
1430 : 0 : NO_END_OF_COPY_GOTO;
1431 : : }
1432 : : }
1433 : :
1434 : : /* Get the next character */
1238 heikki.linnakangas@i 1435 [ - + - - ]:CBC 320 : IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
1436 : : /* if hit_eof, c2 will become '\0' */
1109 1437 : 320 : c2 = copy_input_buf[input_buf_ptr++];
1438 : :
1238 1439 [ + - + + ]: 320 : if (c2 != '\r' && c2 != '\n')
1440 : : {
1441 [ - + ]: 3 : if (!cstate->opts.csv_mode)
1238 heikki.linnakangas@i 1442 [ # # ]:UBC 0 : ereport(ERROR,
1443 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1444 : : errmsg("end-of-copy marker corrupt")));
1445 : : else
1238 heikki.linnakangas@i 1446 :CBC 3 : NO_END_OF_COPY_GOTO;
1447 : : }
1448 : :
1449 [ + + + - ]: 317 : if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
1450 [ - + - - ]: 317 : (cstate->eol_type == EOL_CRNL && c2 != '\n') ||
1451 [ - + - - ]: 317 : (cstate->eol_type == EOL_CR && c2 != '\r'))
1452 : : {
1238 heikki.linnakangas@i 1453 [ # # ]:UBC 0 : ereport(ERROR,
1454 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1455 : : errmsg("end-of-copy marker does not match previous newline style")));
1456 : : }
1457 : :
1458 : : /*
1459 : : * Transfer only the data before the \. into line_buf, then
1460 : : * discard the data and the \. sequence.
1461 : : */
1109 heikki.linnakangas@i 1462 [ - + ]:CBC 317 : if (prev_raw_ptr > cstate->input_buf_index)
1238 heikki.linnakangas@i 1463 :UBC 0 : appendBinaryStringInfo(&cstate->line_buf,
1109 1464 : 0 : cstate->input_buf + cstate->input_buf_index,
1465 : 0 : prev_raw_ptr - cstate->input_buf_index);
1109 heikki.linnakangas@i 1466 :CBC 317 : cstate->input_buf_index = input_buf_ptr;
1238 1467 : 317 : result = true; /* report EOF */
1468 : 317 : break;
1469 : : }
1470 [ + + ]: 3997 : else if (!cstate->opts.csv_mode)
1471 : : {
1472 : : /*
1473 : : * If we are here, it means we found a backslash followed by
1474 : : * something other than a period. In non-CSV mode, anything
1475 : : * after a backslash is special, so we skip over that second
1476 : : * character too. If we didn't do that \\. would be
1477 : : * considered an eof-of copy, while in non-CSV mode it is a
1478 : : * literal backslash followed by a period. In CSV mode,
1479 : : * backslashes are not special, so we want to process the
1480 : : * character after the backslash just like a normal character,
1481 : : * so we don't increment in those cases.
1482 : : */
1109 1483 : 3994 : input_buf_ptr++;
1484 : : }
1485 : : }
1486 : :
1487 : : /*
1488 : : * This label is for CSV cases where \. appears at the start of a
1489 : : * line, but there is more text after it, meaning it was a data value.
1490 : : * We are more strict for \. in CSV mode because \. could be a data
1491 : : * value, while in non-CSV mode, \. cannot be a data value.
1492 : : */
1238 1493 : 12826561 : not_end_of_copy:
1494 : 12830558 : first_char_in_line = false;
1495 : : } /* end of outer loop */
1496 : :
1497 : : /*
1498 : : * Transfer any still-uncopied data to line_buf.
1499 : : */
1500 [ + + ]: 724324 : REFILL_LINEBUF;
1501 : :
1502 : 724324 : return result;
1503 : : }
1504 : :
1505 : : /*
1506 : : * Return decimal value for a hexadecimal digit
1507 : : */
1508 : : static int
1238 heikki.linnakangas@i 1509 :UBC 0 : GetDecimalFromHex(char hex)
1510 : : {
1511 [ # # ]: 0 : if (isdigit((unsigned char) hex))
1512 : 0 : return hex - '0';
1513 : : else
1514 : 0 : return tolower((unsigned char) hex) - 'a' + 10;
1515 : : }
1516 : :
1517 : : /*
1518 : : * Parse the current line into separate attributes (fields),
1519 : : * performing de-escaping as needed.
1520 : : *
1521 : : * The input is in line_buf. We use attribute_buf to hold the result
1522 : : * strings. cstate->raw_fields[k] is set to point to the k'th attribute
1523 : : * string, or NULL when the input matches the null marker string.
1524 : : * This array is expanded as necessary.
1525 : : *
1526 : : * (Note that the caller cannot check for nulls since the returned
1527 : : * string would be the post-de-escaping equivalent, which may look
1528 : : * the same as some valid data string.)
1529 : : *
1530 : : * delim is the column delimiter string (must be just one byte for now).
1531 : : * null_print is the null marker string. Note that this is compared to
1532 : : * the pre-de-escaped input string.
1533 : : *
1534 : : * The return value is the number of fields actually read.
1535 : : */
1536 : : static int
1238 heikki.linnakangas@i 1537 :CBC 723405 : CopyReadAttributesText(CopyFromState cstate)
1538 : : {
1539 : 723405 : char delimc = cstate->opts.delim[0];
1540 : : int fieldno;
1541 : : char *output_ptr;
1542 : : char *cur_ptr;
1543 : : char *line_end_ptr;
1544 : :
1545 : : /*
1546 : : * We need a special case for zero-column tables: check that the input
1547 : : * line is empty, and return.
1548 : : */
1549 [ + + ]: 723405 : if (cstate->max_fields <= 0)
1550 : : {
1551 [ - + ]: 4 : if (cstate->line_buf.len != 0)
1238 heikki.linnakangas@i 1552 [ # # ]:UBC 0 : ereport(ERROR,
1553 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1554 : : errmsg("extra data after last expected column")));
1238 heikki.linnakangas@i 1555 :CBC 4 : return 0;
1556 : : }
1557 : :
1558 : 723401 : resetStringInfo(&cstate->attribute_buf);
1559 : :
1560 : : /*
1561 : : * The de-escaped attributes will certainly not be longer than the input
1562 : : * data line, so we can just force attribute_buf to be large enough and
1563 : : * then transfer data without any checks for enough space. We need to do
1564 : : * it this way because enlarging attribute_buf mid-stream would invalidate
1565 : : * pointers already stored into cstate->raw_fields[].
1566 : : */
1567 [ + + ]: 723401 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1568 : 4 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1569 : 723401 : output_ptr = cstate->attribute_buf.data;
1570 : :
1571 : : /* set pointer variables for loop */
1572 : 723401 : cur_ptr = cstate->line_buf.data;
1573 : 723401 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1574 : :
1575 : : /* Outer loop iterates over fields */
1576 : 723401 : fieldno = 0;
1577 : : for (;;)
1578 : 1585970 : {
1579 : 2309371 : bool found_delim = false;
1580 : : char *start_ptr;
1581 : : char *end_ptr;
1582 : : int input_len;
1583 : 2309371 : bool saw_non_ascii = false;
1584 : :
1585 : : /* Make sure there is enough space for the next value */
1586 [ + + ]: 2309371 : if (fieldno >= cstate->max_fields)
1587 : : {
1588 : 18 : cstate->max_fields *= 2;
1589 : 18 : cstate->raw_fields =
1590 : 18 : repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
1591 : : }
1592 : :
1593 : : /* Remember start of field on both input and output sides */
1594 : 2309371 : start_ptr = cur_ptr;
1595 : 2309371 : cstate->raw_fields[fieldno] = output_ptr;
1596 : :
1597 : : /*
1598 : : * Scan data for field.
1599 : : *
1600 : : * Note that in this loop, we are scanning to locate the end of field
1601 : : * and also speculatively performing de-escaping. Once we find the
1602 : : * end-of-field, we can match the raw field contents against the null
1603 : : * marker string. Only after that comparison fails do we know that
1604 : : * de-escaping is actually the right thing to do; therefore we *must
1605 : : * not* throw any syntax errors before we've done the null-marker
1606 : : * check.
1607 : : */
1608 : : for (;;)
1609 : 11242186 : {
1610 : : char c;
1611 : :
1612 : 13551557 : end_ptr = cur_ptr;
1613 [ + + ]: 13551557 : if (cur_ptr >= line_end_ptr)
1614 : 723398 : break;
1615 : 12828159 : c = *cur_ptr++;
1616 [ + + ]: 12828159 : if (c == delimc)
1617 : : {
1618 : 1585973 : found_delim = true;
1619 : 1585973 : break;
1620 : : }
1621 [ + + ]: 11242186 : if (c == '\\')
1622 : : {
1623 [ - + ]: 3994 : if (cur_ptr >= line_end_ptr)
1238 heikki.linnakangas@i 1624 :UBC 0 : break;
1238 heikki.linnakangas@i 1625 :CBC 3994 : c = *cur_ptr++;
1626 [ + + - - : 3994 : switch (c)
+ - - -
+ ]
1627 : : {
1628 : 6 : case '0':
1629 : : case '1':
1630 : : case '2':
1631 : : case '3':
1632 : : case '4':
1633 : : case '5':
1634 : : case '6':
1635 : : case '7':
1636 : : {
1637 : : /* handle \013 */
1638 : : int val;
1639 : :
1640 : 6 : val = OCTVALUE(c);
1641 [ + + ]: 6 : if (cur_ptr < line_end_ptr)
1642 : : {
1643 : 3 : c = *cur_ptr;
1644 [ - + - - ]: 3 : if (ISOCTAL(c))
1645 : : {
1238 heikki.linnakangas@i 1646 :UBC 0 : cur_ptr++;
1647 : 0 : val = (val << 3) + OCTVALUE(c);
1648 [ # # ]: 0 : if (cur_ptr < line_end_ptr)
1649 : : {
1650 : 0 : c = *cur_ptr;
1651 [ # # # # ]: 0 : if (ISOCTAL(c))
1652 : : {
1653 : 0 : cur_ptr++;
1654 : 0 : val = (val << 3) + OCTVALUE(c);
1655 : : }
1656 : : }
1657 : : }
1658 : : }
1238 heikki.linnakangas@i 1659 :CBC 6 : c = val & 0377;
1660 [ - + - - ]: 6 : if (c == '\0' || IS_HIGHBIT_SET(c))
1661 : 6 : saw_non_ascii = true;
1662 : : }
1663 : 6 : break;
1664 : 6 : case 'x':
1665 : : /* Handle \x3F */
1666 [ + + ]: 6 : if (cur_ptr < line_end_ptr)
1667 : : {
1668 : 3 : char hexchar = *cur_ptr;
1669 : :
1670 [ - + ]: 3 : if (isxdigit((unsigned char) hexchar))
1671 : : {
1238 heikki.linnakangas@i 1672 :UBC 0 : int val = GetDecimalFromHex(hexchar);
1673 : :
1674 : 0 : cur_ptr++;
1675 [ # # ]: 0 : if (cur_ptr < line_end_ptr)
1676 : : {
1677 : 0 : hexchar = *cur_ptr;
1678 [ # # ]: 0 : if (isxdigit((unsigned char) hexchar))
1679 : : {
1680 : 0 : cur_ptr++;
1681 : 0 : val = (val << 4) + GetDecimalFromHex(hexchar);
1682 : : }
1683 : : }
1684 : 0 : c = val & 0xff;
1685 [ # # # # ]: 0 : if (c == '\0' || IS_HIGHBIT_SET(c))
1686 : 0 : saw_non_ascii = true;
1687 : : }
1688 : : }
1238 heikki.linnakangas@i 1689 :CBC 6 : break;
1238 heikki.linnakangas@i 1690 :UBC 0 : case 'b':
1691 : 0 : c = '\b';
1692 : 0 : break;
1693 : 0 : case 'f':
1694 : 0 : c = '\f';
1695 : 0 : break;
1238 heikki.linnakangas@i 1696 :CBC 1525 : case 'n':
1697 : 1525 : c = '\n';
1698 : 1525 : break;
1238 heikki.linnakangas@i 1699 :UBC 0 : case 'r':
1700 : 0 : c = '\r';
1701 : 0 : break;
1702 : 0 : case 't':
1703 : 0 : c = '\t';
1704 : 0 : break;
1705 : 0 : case 'v':
1706 : 0 : c = '\v';
1707 : 0 : break;
1708 : :
1709 : : /*
1710 : : * in all other cases, take the char after '\'
1711 : : * literally
1712 : : */
1713 : : }
1714 : : }
1715 : :
1716 : : /* Add c to output string */
1238 heikki.linnakangas@i 1717 :CBC 11242186 : *output_ptr++ = c;
1718 : : }
1719 : :
1720 : : /* Check whether raw input matched null marker */
1721 : 2309371 : input_len = end_ptr - start_ptr;
1722 [ + + ]: 2309371 : if (input_len == cstate->opts.null_print_len &&
1723 [ + + ]: 120342 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
1724 : 2401 : cstate->raw_fields[fieldno] = NULL;
1725 : : /* Check whether raw input matched default marker */
396 andrew@dunslane.net 1726 [ + + ]: 2306970 : else if (fieldno < list_length(cstate->attnumlist) &&
1727 [ + + ]: 2306949 : cstate->opts.default_print &&
398 1728 [ + + ]: 57 : input_len == cstate->opts.default_print_len &&
1729 [ + - ]: 15 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
1730 : 12 : {
1731 : : /* fieldno is 0-indexed and attnum is 1-indexed */
1732 : 15 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
1733 : :
1734 [ + + ]: 15 : if (cstate->defexprs[m] != NULL)
1735 : : {
1736 : : /* defaults contain entries for all physical attributes */
1737 : 12 : cstate->defaults[m] = true;
1738 : : }
1739 : : else
1740 : : {
1741 : 3 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
1742 : 3 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1743 : :
1744 [ + - ]: 3 : ereport(ERROR,
1745 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1746 : : errmsg("unexpected default marker in COPY data"),
1747 : : errdetail("Column \"%s\" has no default value.",
1748 : : NameStr(att->attname))));
1749 : : }
1750 : : }
1751 : : else
1752 : : {
1753 : : /*
1754 : : * At this point we know the field is supposed to contain data.
1755 : : *
1756 : : * If we de-escaped any non-7-bit-ASCII chars, make sure the
1757 : : * resulting string is valid data for the db encoding.
1758 : : */
1238 heikki.linnakangas@i 1759 [ - + ]: 2306955 : if (saw_non_ascii)
1760 : : {
1238 heikki.linnakangas@i 1761 :UBC 0 : char *fld = cstate->raw_fields[fieldno];
1762 : :
1763 : 0 : pg_verifymbstr(fld, output_ptr - fld, false);
1764 : : }
1765 : : }
1766 : :
1767 : : /* Terminate attribute value in output area */
1238 heikki.linnakangas@i 1768 :CBC 2309368 : *output_ptr++ = '\0';
1769 : :
1770 : 2309368 : fieldno++;
1771 : : /* Done if we hit EOL instead of a delim */
1772 [ + + ]: 2309368 : if (!found_delim)
1773 : 723398 : break;
1774 : : }
1775 : :
1776 : : /* Clean up state of attribute_buf */
1777 : 723398 : output_ptr--;
1778 [ - + ]: 723398 : Assert(*output_ptr == '\0');
1779 : 723398 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
1780 : :
1781 : 723398 : return fieldno;
1782 : : }
1783 : :
1784 : : /*
1785 : : * Parse the current line into separate attributes (fields),
1786 : : * performing de-escaping as needed. This has exactly the same API as
1787 : : * CopyReadAttributesText, except we parse the fields according to
1788 : : * "standard" (i.e. common) CSV usage.
1789 : : */
1790 : : static int
1791 : 202 : CopyReadAttributesCSV(CopyFromState cstate)
1792 : : {
1793 : 202 : char delimc = cstate->opts.delim[0];
1794 : 202 : char quotec = cstate->opts.quote[0];
1795 : 202 : char escapec = cstate->opts.escape[0];
1796 : : int fieldno;
1797 : : char *output_ptr;
1798 : : char *cur_ptr;
1799 : : char *line_end_ptr;
1800 : :
1801 : : /*
1802 : : * We need a special case for zero-column tables: check that the input
1803 : : * line is empty, and return.
1804 : : */
1805 [ - + ]: 202 : if (cstate->max_fields <= 0)
1806 : : {
1238 heikki.linnakangas@i 1807 [ # # ]:UBC 0 : if (cstate->line_buf.len != 0)
1808 [ # # ]: 0 : ereport(ERROR,
1809 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1810 : : errmsg("extra data after last expected column")));
1811 : 0 : return 0;
1812 : : }
1813 : :
1238 heikki.linnakangas@i 1814 :CBC 202 : resetStringInfo(&cstate->attribute_buf);
1815 : :
1816 : : /*
1817 : : * The de-escaped attributes will certainly not be longer than the input
1818 : : * data line, so we can just force attribute_buf to be large enough and
1819 : : * then transfer data without any checks for enough space. We need to do
1820 : : * it this way because enlarging attribute_buf mid-stream would invalidate
1821 : : * pointers already stored into cstate->raw_fields[].
1822 : : */
1823 [ - + ]: 202 : if (cstate->attribute_buf.maxlen <= cstate->line_buf.len)
1238 heikki.linnakangas@i 1824 :UBC 0 : enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len);
1238 heikki.linnakangas@i 1825 :CBC 202 : output_ptr = cstate->attribute_buf.data;
1826 : :
1827 : : /* set pointer variables for loop */
1828 : 202 : cur_ptr = cstate->line_buf.data;
1829 : 202 : line_end_ptr = cstate->line_buf.data + cstate->line_buf.len;
1830 : :
1831 : : /* Outer loop iterates over fields */
1832 : 202 : fieldno = 0;
1833 : : for (;;)
1834 : 245 : {
1835 : 447 : bool found_delim = false;
1836 : 447 : bool saw_quote = false;
1837 : : char *start_ptr;
1838 : : char *end_ptr;
1839 : : int input_len;
1840 : :
1841 : : /* Make sure there is enough space for the next value */
1842 [ - + ]: 447 : if (fieldno >= cstate->max_fields)
1843 : : {
1238 heikki.linnakangas@i 1844 :UBC 0 : cstate->max_fields *= 2;
1845 : 0 : cstate->raw_fields =
1846 : 0 : repalloc(cstate->raw_fields, cstate->max_fields * sizeof(char *));
1847 : : }
1848 : :
1849 : : /* Remember start of field on both input and output sides */
1238 heikki.linnakangas@i 1850 :CBC 447 : start_ptr = cur_ptr;
1851 : 447 : cstate->raw_fields[fieldno] = output_ptr;
1852 : :
1853 : : /*
1854 : : * Scan data for field,
1855 : : *
1856 : : * The loop starts in "not quote" mode and then toggles between that
1857 : : * and "in quote" mode. The loop exits normally if it is in "not
1858 : : * quote" mode and a delimiter or line end is seen.
1859 : : */
1860 : : for (;;)
1861 : 91 : {
1862 : : char c;
1863 : :
1864 : : /* Not in quote */
1865 : : for (;;)
1866 : : {
1867 : 1433 : end_ptr = cur_ptr;
1868 [ + + ]: 1433 : if (cur_ptr >= line_end_ptr)
1869 : 199 : goto endfield;
1870 : 1234 : c = *cur_ptr++;
1871 : : /* unquoted field delimiter */
1872 [ + + ]: 1234 : if (c == delimc)
1873 : : {
1874 : 248 : found_delim = true;
1875 : 248 : goto endfield;
1876 : : }
1877 : : /* start of quoted field (or part of field) */
1878 [ + + ]: 986 : if (c == quotec)
1879 : : {
1880 : 91 : saw_quote = true;
1881 : 91 : break;
1882 : : }
1883 : : /* Add c to output string */
1884 : 895 : *output_ptr++ = c;
1885 : : }
1886 : :
1887 : : /* In quote */
1888 : : for (;;)
1889 : : {
1890 : 585 : end_ptr = cur_ptr;
1891 [ - + ]: 585 : if (cur_ptr >= line_end_ptr)
1238 heikki.linnakangas@i 1892 [ # # ]:UBC 0 : ereport(ERROR,
1893 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1894 : : errmsg("unterminated CSV quoted field")));
1895 : :
1238 heikki.linnakangas@i 1896 :CBC 585 : c = *cur_ptr++;
1897 : :
1898 : : /* escape within a quoted field */
1899 [ + + ]: 585 : if (c == escapec)
1900 : : {
1901 : : /*
1902 : : * peek at the next char if available, and escape it if it
1903 : : * is an escape char or a quote char
1904 : : */
1905 [ + + ]: 59 : if (cur_ptr < line_end_ptr)
1906 : : {
1907 : 36 : char nextc = *cur_ptr;
1908 : :
1909 [ + + - + ]: 36 : if (nextc == escapec || nextc == quotec)
1910 : : {
1911 : 12 : *output_ptr++ = nextc;
1912 : 12 : cur_ptr++;
1913 : 12 : continue;
1914 : : }
1915 : : }
1916 : : }
1917 : :
1918 : : /*
1919 : : * end of quoted field. Must do this test after testing for
1920 : : * escape in case quote char and escape char are the same
1921 : : * (which is the common case).
1922 : : */
1923 [ + + ]: 573 : if (c == quotec)
1924 : 91 : break;
1925 : :
1926 : : /* Add c to output string */
1927 : 482 : *output_ptr++ = c;
1928 : : }
1929 : : }
1930 : 447 : endfield:
1931 : :
1932 : : /* Terminate attribute value in output area */
1933 : 447 : *output_ptr++ = '\0';
1934 : :
1935 : : /* Check whether raw input matched null marker */
1936 : 447 : input_len = end_ptr - start_ptr;
1937 [ + + + + ]: 447 : if (!saw_quote && input_len == cstate->opts.null_print_len &&
1938 [ + - ]: 22 : strncmp(start_ptr, cstate->opts.null_print, input_len) == 0)
1939 : 22 : cstate->raw_fields[fieldno] = NULL;
1940 : : /* Check whether raw input matched default marker */
396 andrew@dunslane.net 1941 [ + - ]: 425 : else if (fieldno < list_length(cstate->attnumlist) &&
1942 [ + + ]: 425 : cstate->opts.default_print &&
398 1943 [ + + ]: 75 : input_len == cstate->opts.default_print_len &&
1944 [ + - ]: 21 : strncmp(start_ptr, cstate->opts.default_print, input_len) == 0)
1945 : : {
1946 : : /* fieldno is 0-index and attnum is 1-index */
1947 : 21 : int m = list_nth_int(cstate->attnumlist, fieldno) - 1;
1948 : :
1949 [ + + ]: 21 : if (cstate->defexprs[m] != NULL)
1950 : : {
1951 : : /* defaults contain entries for all physical attributes */
1952 : 18 : cstate->defaults[m] = true;
1953 : : }
1954 : : else
1955 : : {
1956 : 3 : TupleDesc tupDesc = RelationGetDescr(cstate->rel);
1957 : 3 : Form_pg_attribute att = TupleDescAttr(tupDesc, m);
1958 : :
1959 [ + - ]: 3 : ereport(ERROR,
1960 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1961 : : errmsg("unexpected default marker in COPY data"),
1962 : : errdetail("Column \"%s\" has no default value.",
1963 : : NameStr(att->attname))));
1964 : : }
1965 : : }
1966 : :
1238 heikki.linnakangas@i 1967 : 444 : fieldno++;
1968 : : /* Done if we hit EOL instead of a delim */
1969 [ + + ]: 444 : if (!found_delim)
1970 : 199 : break;
1971 : : }
1972 : :
1973 : : /* Clean up state of attribute_buf */
1974 : 199 : output_ptr--;
1975 [ - + ]: 199 : Assert(*output_ptr == '\0');
1976 : 199 : cstate->attribute_buf.len = (output_ptr - cstate->attribute_buf.data);
1977 : :
1978 : 199 : return fieldno;
1979 : : }
1980 : :
1981 : :
1982 : : /*
1983 : : * Read a binary attribute
1984 : : */
1985 : : static Datum
1986 : 79 : CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
1987 : : Oid typioparam, int32 typmod,
1988 : : bool *isnull)
1989 : : {
1990 : : int32 fld_size;
1991 : : Datum result;
1992 : :
1993 [ - + ]: 79 : if (!CopyGetInt32(cstate, &fld_size))
1238 heikki.linnakangas@i 1994 [ # # ]:UBC 0 : ereport(ERROR,
1995 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
1996 : : errmsg("unexpected EOF in COPY data")));
1238 heikki.linnakangas@i 1997 [ + + ]:CBC 79 : if (fld_size == -1)
1998 : : {
1999 : 15 : *isnull = true;
2000 : 15 : return ReceiveFunctionCall(flinfo, NULL, typioparam, typmod);
2001 : : }
2002 [ - + ]: 64 : if (fld_size < 0)
1238 heikki.linnakangas@i 2003 [ # # ]:UBC 0 : ereport(ERROR,
2004 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2005 : : errmsg("invalid field size")));
2006 : :
2007 : : /* reset attribute_buf to empty, and load raw data in it */
1238 heikki.linnakangas@i 2008 :CBC 64 : resetStringInfo(&cstate->attribute_buf);
2009 : :
2010 : 64 : enlargeStringInfo(&cstate->attribute_buf, fld_size);
2011 : 64 : if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
2012 [ - + ]: 64 : fld_size) != fld_size)
1238 heikki.linnakangas@i 2013 [ # # ]:UBC 0 : ereport(ERROR,
2014 : : (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2015 : : errmsg("unexpected EOF in COPY data")));
2016 : :
1238 heikki.linnakangas@i 2017 :CBC 64 : cstate->attribute_buf.len = fld_size;
2018 : 64 : cstate->attribute_buf.data[fld_size] = '\0';
2019 : :
2020 : : /* Call the column type's binary input converter */
2021 : 64 : result = ReceiveFunctionCall(flinfo, &cstate->attribute_buf,
2022 : : typioparam, typmod);
2023 : :
2024 : : /* Trouble if it didn't eat the whole buffer */
2025 [ + + ]: 64 : if (cstate->attribute_buf.cursor != cstate->attribute_buf.len)
2026 [ + - ]: 1 : ereport(ERROR,
2027 : : (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
2028 : : errmsg("incorrect binary data format")));
2029 : :
2030 : 63 : *isnull = false;
2031 : 63 : return result;
2032 : : }
|
