TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_backup_tar.c
4 : *
5 : * This file is copied from the 'files' format file, but dumps data into
6 : * one temp file then sends it to the output TAR archive.
7 : *
8 : * The tar format also includes a 'restore.sql' script which is there for
9 : * the benefit of humans. This script is never used by pg_restore.
10 : *
11 : * NOTE: If you untar the created 'tar' file, the resulting files are
12 : * compatible with the 'directory' format. Please keep the two formats in
13 : * sync.
14 : *
15 : * See the headers to pg_backup_directory & pg_restore for more details.
16 : *
17 : * Copyright (c) 2000, Philip Warner
18 : * Rights are granted to use this software in any way so long
19 : * as this notice is not removed.
20 : *
21 : * The author is not responsible for loss or damages that may
22 : * result from its use.
23 : *
24 : *
25 : * IDENTIFICATION
26 : * src/bin/pg_dump/pg_backup_tar.c
27 : *
28 : *-------------------------------------------------------------------------
29 : */
30 : #include "postgres_fe.h"
31 :
32 : #include <sys/stat.h>
33 : #include <ctype.h>
34 : #include <limits.h>
35 : #include <unistd.h>
36 :
37 : #include "common/file_utils.h"
38 : #include "fe_utils/string_utils.h"
39 : #include "pg_backup_archiver.h"
40 : #include "pg_backup_tar.h"
41 : #include "pg_backup_utils.h"
42 : #include "pgtar.h"
43 :
44 : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 : static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 : static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 : static int _WriteByte(ArchiveHandle *AH, const int i);
49 : static int _ReadByte(ArchiveHandle *AH);
50 : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 : static void _CloseArchive(ArchiveHandle *AH);
53 : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 :
58 : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
59 : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
62 :
63 : #define K_STD_BUF_SIZE 1024
64 :
65 :
66 : typedef struct
67 : {
68 : FILE *nFH;
69 : FILE *tarFH;
70 : FILE *tmpFH;
71 : char *targetFile;
72 : char mode;
73 : pgoff_t pos;
74 : pgoff_t fileLen;
75 : ArchiveHandle *AH;
76 : } TAR_MEMBER;
77 :
78 : typedef struct
79 : {
80 : int hasSeek;
81 : pgoff_t filePos;
82 : TAR_MEMBER *loToc;
83 : FILE *tarFH;
84 : pgoff_t tarFHpos;
85 : pgoff_t tarNextMember;
86 : TAR_MEMBER *FH;
87 : int isSpecialScript;
88 : TAR_MEMBER *scriptTH;
89 : } lclContext;
90 :
91 : typedef struct
92 : {
93 : TAR_MEMBER *TH;
94 : char *filename;
95 : } lclTocEntry;
96 :
97 : static void _LoadLOs(ArchiveHandle *AH);
98 :
99 : static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
100 : static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th);
101 :
102 : #ifdef __NOT_USED__
103 : static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
104 : #endif
105 : static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3);
106 :
107 : static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
108 : static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
109 : static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
110 : static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
111 : static void _tarWriteHeader(TAR_MEMBER *th);
112 : static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
113 : static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
114 :
115 : static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
116 :
117 : /*
118 : * Initializer
119 : */
120 : void
121 CBC 5 : InitArchiveFmt_Tar(ArchiveHandle *AH)
122 : {
123 : lclContext *ctx;
124 :
125 : /* Assuming static functions, this can be copied for each format. */
126 5 : AH->ArchiveEntryPtr = _ArchiveEntry;
127 5 : AH->StartDataPtr = _StartData;
128 5 : AH->WriteDataPtr = _WriteData;
129 5 : AH->EndDataPtr = _EndData;
130 5 : AH->WriteBytePtr = _WriteByte;
131 5 : AH->ReadBytePtr = _ReadByte;
132 5 : AH->WriteBufPtr = _WriteBuf;
133 5 : AH->ReadBufPtr = _ReadBuf;
134 5 : AH->ClosePtr = _CloseArchive;
135 5 : AH->ReopenPtr = NULL;
136 5 : AH->PrintTocDataPtr = _PrintTocData;
137 5 : AH->ReadExtraTocPtr = _ReadExtraToc;
138 5 : AH->WriteExtraTocPtr = _WriteExtraToc;
139 5 : AH->PrintExtraTocPtr = _PrintExtraToc;
140 :
141 GNC 5 : AH->StartLOsPtr = _StartLOs;
142 5 : AH->StartLOPtr = _StartLO;
143 5 : AH->EndLOPtr = _EndLO;
144 5 : AH->EndLOsPtr = _EndLOs;
145 CBC 5 : AH->ClonePtr = NULL;
146 5 : AH->DeClonePtr = NULL;
147 :
148 5 : AH->WorkerJobDumpPtr = NULL;
149 5 : AH->WorkerJobRestorePtr = NULL;
150 :
151 : /*
152 : * Set up some special context used in compressing data.
153 : */
154 GNC 5 : ctx = pg_malloc0_object(lclContext);
155 CBC 5 : AH->formatData = (void *) ctx;
156 5 : ctx->filePos = 0;
157 5 : ctx->isSpecialScript = 0;
158 :
159 : /* Initialize LO buffering */
160 5 : AH->lo_buf_size = LOBBUFSIZE;
161 5 : AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
162 :
163 : /*
164 : * Now open the tar file, and load the TOC if we're in read mode.
165 : */
166 5 : if (AH->mode == archModeWrite)
167 : {
168 3 : if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
169 : {
170 2 : ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
171 2 : if (ctx->tarFH == NULL)
172 UBC 0 : pg_fatal("could not open TOC file \"%s\" for output: %m",
173 : AH->fSpec);
174 : }
175 : else
176 : {
177 CBC 1 : ctx->tarFH = stdout;
178 1 : if (ctx->tarFH == NULL)
179 UBC 0 : pg_fatal("could not open TOC file for output: %m");
180 : }
181 :
182 CBC 3 : ctx->tarFHpos = 0;
183 :
184 : /*
185 : * Make unbuffered since we will dup() it, and the buffers screw each
186 : * other
187 : */
188 : /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
189 :
190 3 : ctx->hasSeek = checkSeek(ctx->tarFH);
191 :
192 : /*
193 : * We don't support compression because reading the files back is not
194 : * possible since gzdopen uses buffered IO which totally screws file
195 : * positioning.
196 : */
197 GNC 3 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
198 CBC 1 : pg_fatal("compression is not supported by tar archive format");
199 : }
200 : else
201 : { /* Read Mode */
202 2 : if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
203 : {
204 2 : ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
205 2 : if (ctx->tarFH == NULL)
206 UBC 0 : pg_fatal("could not open TOC file \"%s\" for input: %m",
207 : AH->fSpec);
208 : }
209 : else
210 : {
211 0 : ctx->tarFH = stdin;
212 0 : if (ctx->tarFH == NULL)
213 0 : pg_fatal("could not open TOC file for input: %m");
214 : }
215 :
216 : /*
217 : * Make unbuffered since we will dup() it, and the buffers screw each
218 : * other
219 : */
220 : /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
221 :
222 CBC 2 : ctx->tarFHpos = 0;
223 :
224 2 : ctx->hasSeek = checkSeek(ctx->tarFH);
225 :
226 2 : ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
227 2 : ReadHead(AH);
228 2 : ReadToc(AH);
229 2 : tarClose(AH, ctx->FH); /* Nothing else in the file... */
230 : }
231 4 : }
232 :
233 : /*
234 : * - Start a new TOC entry
235 : * Setup the output file name.
236 : */
237 : static void
238 282 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
239 : {
240 : lclTocEntry *ctx;
241 : char fn[K_STD_BUF_SIZE];
242 :
243 GNC 282 : ctx = pg_malloc0_object(lclTocEntry);
244 CBC 282 : if (te->dataDumper != NULL)
245 : {
246 28 : snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
247 28 : ctx->filename = pg_strdup(fn);
248 : }
249 : else
250 : {
251 254 : ctx->filename = NULL;
252 254 : ctx->TH = NULL;
253 : }
254 282 : te->formatData = (void *) ctx;
255 282 : }
256 :
257 : static void
258 282 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
259 : {
260 282 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
261 :
262 282 : if (ctx->filename)
263 28 : WriteStr(AH, ctx->filename);
264 : else
265 254 : WriteStr(AH, "");
266 282 : }
267 :
268 : static void
269 282 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
270 : {
271 282 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
272 :
273 282 : if (ctx == NULL)
274 : {
275 GNC 282 : ctx = pg_malloc0_object(lclTocEntry);
276 CBC 282 : te->formatData = (void *) ctx;
277 : }
278 :
279 282 : ctx->filename = ReadStr(AH);
280 282 : if (strlen(ctx->filename) == 0)
281 : {
282 254 : free(ctx->filename);
283 254 : ctx->filename = NULL;
284 : }
285 282 : ctx->TH = NULL;
286 282 : }
287 :
288 : static void
289 548 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
290 : {
291 548 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
292 :
293 548 : if (AH->public.verbose && ctx->filename != NULL)
294 UBC 0 : ahprintf(AH, "-- File: %s\n", ctx->filename);
295 CBC 548 : }
296 :
297 : static void
298 27 : _StartData(ArchiveHandle *AH, TocEntry *te)
299 : {
300 27 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
301 :
302 27 : tctx->TH = tarOpen(AH, tctx->filename, 'w');
303 27 : }
304 :
305 : static TAR_MEMBER *
306 66 : tarOpen(ArchiveHandle *AH, const char *filename, char mode)
307 : {
308 66 : lclContext *ctx = (lclContext *) AH->formatData;
309 : TAR_MEMBER *tm;
310 :
311 66 : if (mode == 'r')
312 : {
313 32 : tm = _tarPositionTo(AH, filename);
314 32 : if (!tm) /* Not found */
315 : {
316 UBC 0 : if (filename)
317 : {
318 : /*
319 : * Couldn't find the requested file. Future: do SEEK(0) and
320 : * retry.
321 : */
322 0 : pg_fatal("could not find file \"%s\" in archive", filename);
323 : }
324 : else
325 : {
326 : /* Any file OK, none left, so return NULL */
327 0 : return NULL;
328 : }
329 : }
330 :
331 GNC 32 : if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
332 CBC 32 : tm->nFH = ctx->tarFH;
333 : else
334 UBC 0 : pg_fatal("compression is not supported by tar archive format");
335 : }
336 : else
337 : {
338 : int old_umask;
339 :
340 GNC 34 : tm = pg_malloc0_object(TAR_MEMBER);
341 :
342 : /*
343 : * POSIX does not require, but permits, tmpfile() to restrict file
344 : * permissions. Given an OS crash after we write data, the filesystem
345 : * might retain the data but forget tmpfile()'s unlink(). If so, the
346 : * file mode protects confidentiality of the data written.
347 : */
348 CBC 34 : old_umask = umask(S_IRWXG | S_IRWXO);
349 :
350 : #ifndef WIN32
351 34 : tm->tmpFH = tmpfile();
352 : #else
353 :
354 : /*
355 : * On WIN32, tmpfile() generates a filename in the root directory,
356 : * which requires administrative permissions on certain systems. Loop
357 : * until we find a unique file name we can create.
358 : */
359 : while (1)
360 : {
361 : char *name;
362 : int fd;
363 :
364 : name = _tempnam(NULL, "pg_temp_");
365 : if (name == NULL)
366 : break;
367 : fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
368 : O_TEMPORARY, S_IRUSR | S_IWUSR);
369 : free(name);
370 :
371 : if (fd != -1) /* created a file */
372 : {
373 : tm->tmpFH = fdopen(fd, "w+b");
374 : break;
375 : }
376 : else if (errno != EEXIST) /* failure other than file exists */
377 : break;
378 : }
379 : #endif
380 :
381 34 : if (tm->tmpFH == NULL)
382 UBC 0 : pg_fatal("could not generate temporary file name: %m");
383 :
384 CBC 34 : umask(old_umask);
385 :
386 GNC 34 : if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
387 CBC 34 : tm->nFH = tm->tmpFH;
388 : else
389 UBC 0 : pg_fatal("compression is not supported by tar archive format");
390 :
391 CBC 34 : tm->AH = AH;
392 34 : tm->targetFile = pg_strdup(filename);
393 : }
394 :
395 66 : tm->mode = mode;
396 66 : tm->tarFH = ctx->tarFH;
397 :
398 66 : return tm;
399 : }
400 :
401 : static void
402 66 : tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
403 : {
404 GNC 66 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
405 UBC 0 : pg_fatal("compression is not supported by tar archive format");
406 :
407 CBC 66 : if (th->mode == 'w')
408 34 : _tarAddFile(AH, th); /* This will close the temp file */
409 :
410 : /*
411 : * else Nothing to do for normal read since we don't dup() normal file
412 : * handle, and we don't use temp files.
413 : */
414 :
415 GNC 66 : free(th->targetFile);
416 ECB :
417 CBC 66 : th->nFH = NULL;
418 GIC 66 : }
419 :
420 : #ifdef __NOT_USED__
421 : static char *
422 : tarGets(char *buf, size_t len, TAR_MEMBER *th)
423 : {
424 : char *s;
425 : size_t cnt = 0;
426 : char c = ' ';
427 : int eof = 0;
428 :
429 : /* Can't read past logical EOF */
430 : if (len > (th->fileLen - th->pos))
431 : len = th->fileLen - th->pos;
432 :
433 : while (cnt < len && c != '\n')
434 : {
435 : if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
436 : {
437 : eof = 1;
438 : break;
439 : }
440 : buf[cnt++] = c;
441 : }
442 :
443 : if (eof && cnt == 0)
444 : s = NULL;
445 : else
446 : {
447 : buf[cnt++] = '\0';
448 : s = buf;
449 : }
450 :
451 : if (s)
452 : {
453 : len = strlen(s);
454 : th->pos += len;
455 : }
456 :
457 : return s;
458 : }
459 : #endif
460 :
461 : /*
462 : * Just read bytes from the archive. This is the low level read routine
463 : * that is used for ALL reads on a tar file.
464 : */
465 ECB : static size_t
466 GIC 42871 : _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
467 ECB : {
468 GIC 42871 : lclContext *ctx = (lclContext *) AH->formatData;
469 ECB : size_t avail;
470 CBC 42871 : size_t used = 0;
471 GIC 42871 : size_t res = 0;
472 ECB :
473 GIC 42871 : Assert(th || fh);
474 ECB :
475 CBC 42871 : avail = AH->lookaheadLen - AH->lookaheadPos;
476 GIC 42871 : if (avail > 0)
477 : {
478 EUB : /* We have some lookahead bytes to use */
479 UBC 0 : if (avail >= len) /* Just use the lookahead buffer */
480 UIC 0 : used = len;
481 EUB : else
482 UIC 0 : used = avail;
483 :
484 EUB : /* Copy, and adjust buffer pos */
485 UBC 0 : memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
486 UIC 0 : AH->lookaheadPos += used;
487 :
488 EUB : /* Adjust required length */
489 UIC 0 : len -= used;
490 : }
491 :
492 ECB : /* Read the file if len > 0 */
493 GIC 42871 : if (len > 0)
494 ECB : {
495 GIC 42871 : if (fh)
496 ECB : {
497 CBC 13763 : res = fread(&((char *) buf)[used], 1, len, fh);
498 GBC 13763 : if (res != len && !feof(fh))
499 UIC 0 : READ_ERROR_EXIT(fh);
500 ECB : }
501 GIC 29108 : else if (th)
502 ECB : {
503 CBC 29108 : res = fread(&((char *) buf)[used], 1, len, th->nFH);
504 GBC 29108 : if (res != len && !feof(th->nFH))
505 UIC 0 : READ_ERROR_EXIT(th->nFH);
506 : }
507 : }
508 ECB :
509 GIC 42871 : ctx->tarFHpos += res + used;
510 ECB :
511 GIC 42871 : return (res + used);
512 : }
513 :
514 ECB : static size_t
515 GIC 29451 : tarRead(void *buf, size_t len, TAR_MEMBER *th)
516 : {
517 : size_t res;
518 ECB :
519 CBC 29451 : if (th->pos + len > th->fileLen)
520 GIC 57 : len = th->fileLen - th->pos;
521 ECB :
522 CBC 29451 : if (len <= 0)
523 GIC 343 : return 0;
524 ECB :
525 GIC 29108 : res = _tarReadRaw(th->AH, buf, len, th, NULL);
526 ECB :
527 GIC 29108 : th->pos += res;
528 ECB :
529 GIC 29108 : return res;
530 : }
531 :
532 ECB : static size_t
533 GIC 31071 : tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
534 : {
535 : size_t res;
536 ECB :
537 GIC 31071 : res = fwrite(buf, 1, len, th->nFH);
538 ECB :
539 CBC 31071 : th->pos += res;
540 GIC 31071 : return res;
541 : }
542 :
543 ECB : static void
544 GIC 59 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
545 ECB : {
546 GIC 59 : lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
547 ECB :
548 GBC 59 : if (tarWrite(data, dLen, tctx->TH) != dLen)
549 LBC 0 : WRITE_ERROR_EXIT;
550 GIC 59 : }
551 :
552 ECB : static void
553 GIC 27 : _EndData(ArchiveHandle *AH, TocEntry *te)
554 ECB : {
555 GIC 27 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
556 :
557 ECB : /* Close the file */
558 CBC 27 : tarClose(AH, tctx->TH);
559 27 : tctx->TH = NULL;
560 GIC 27 : }
561 :
562 : /*
563 : * Print data for a given file
564 : */
565 ECB : static void
566 GIC 27 : _PrintFileData(ArchiveHandle *AH, char *filename)
567 ECB : {
568 GIC 27 : lclContext *ctx = (lclContext *) AH->formatData;
569 : char buf[4096];
570 : size_t cnt;
571 : TAR_MEMBER *th;
572 ECB :
573 GBC 27 : if (!filename)
574 UIC 0 : return;
575 ECB :
576 CBC 27 : th = tarOpen(AH, filename, 'r');
577 GIC 27 : ctx->FH = th;
578 ECB :
579 GIC 54 : while ((cnt = tarRead(buf, 4095, th)) > 0)
580 ECB : {
581 CBC 27 : buf[cnt] = '\0';
582 GIC 27 : ahwrite(buf, 1, cnt, AH);
583 : }
584 ECB :
585 GIC 27 : tarClose(AH, th);
586 : }
587 :
588 :
589 : /*
590 : * Print data for a given TOC entry
591 : */
592 ECB : static void
593 GIC 56 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
594 ECB : {
595 CBC 56 : lclContext *ctx = (lclContext *) AH->formatData;
596 GIC 56 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
597 : int pos1;
598 ECB :
599 GBC 56 : if (!tctx->filename)
600 UIC 0 : return;
601 :
602 : /*
603 : * If we're writing the special restore.sql script, emit a suitable
604 : * command to include each table's data from the corresponding file.
605 : *
606 : * In the COPY case this is a bit klugy because the regular COPY command
607 : * was already printed before we get control.
608 ECB : */
609 GIC 56 : if (ctx->isSpecialScript)
610 ECB : {
611 GIC 28 : if (te->copyStmt)
612 : {
613 ECB : /* Abort the COPY FROM stdin */
614 GIC 27 : ahprintf(AH, "\\.\n");
615 :
616 : /*
617 : * The COPY statement should look like "COPY ... FROM stdin;\n",
618 : * see dumpTableData().
619 ECB : */
620 CBC 27 : pos1 = (int) strlen(te->copyStmt) - 13;
621 27 : if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
622 GBC 27 : strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
623 UIC 0 : pg_fatal("unexpected COPY statement syntax: \"%s\"",
624 : te->copyStmt);
625 :
626 ECB : /* Emit all but the FROM part ... */
627 GIC 27 : ahwrite(te->copyStmt, 1, pos1, AH);
628 ECB : /* ... and insert modified FROM */
629 GIC 27 : ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
630 : }
631 : else
632 : {
633 ECB : /* --inserts mode, no worries, just include the data file */
634 GIC 1 : ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
635 : }
636 ECB :
637 GIC 28 : return;
638 : }
639 ECB :
640 CBC 28 : if (strcmp(te->desc, "BLOBS") == 0)
641 GNC 1 : _LoadLOs(AH);
642 ECB : else
643 GIC 27 : _PrintFileData(AH, tctx->filename);
644 : }
645 :
646 ECB : static void
647 GNC 1 : _LoadLOs(ArchiveHandle *AH)
648 : {
649 ECB : Oid oid;
650 GIC 1 : lclContext *ctx = (lclContext *) AH->formatData;
651 : TAR_MEMBER *th;
652 ECB : size_t cnt;
653 GNC 1 : bool foundLO = false;
654 : char buf[4096];
655 ECB :
656 GNC 1 : StartRestoreLOs(AH);
657 ECB :
658 CBC 1 : th = tarOpen(AH, NULL, 'r'); /* Open next file */
659 GIC 3 : while (th != NULL)
660 ECB : {
661 GIC 3 : ctx->FH = th;
662 ECB :
663 GIC 3 : if (strncmp(th->targetFile, "blob_", 5) == 0)
664 ECB : {
665 CBC 2 : oid = atooid(&th->targetFile[5]);
666 GIC 2 : if (oid != 0)
667 ECB : {
668 GIC 2 : pg_log_info("restoring large object with OID %u", oid);
669 ECB :
670 GNC 2 : StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
671 ECB :
672 GIC 3 : while ((cnt = tarRead(buf, 4095, th)) > 0)
673 ECB : {
674 CBC 1 : buf[cnt] = '\0';
675 GIC 1 : ahwrite(buf, 1, cnt, AH);
676 ECB : }
677 GNC 2 : EndRestoreLO(AH, oid);
678 2 : foundLO = true;
679 ECB : }
680 GIC 2 : tarClose(AH, th);
681 : }
682 : else
683 ECB : {
684 GIC 1 : tarClose(AH, th);
685 :
686 : /*
687 : * Once we have found the first LO, stop at the first non-LO
688 : * entry (which will be 'blobs.toc'). This coding would eat all
689 : * the rest of the archive if there are no LOs ... but this
690 : * function shouldn't be called at all in that case.
691 ECB : */
692 GNC 1 : if (foundLO)
693 GIC 1 : break;
694 : }
695 ECB :
696 GIC 2 : th = tarOpen(AH, NULL, 'r');
697 ECB : }
698 GNC 1 : EndRestoreLOs(AH);
699 GIC 1 : }
700 :
701 :
702 ECB : static int
703 GIC 26259 : _WriteByte(ArchiveHandle *AH, const int i)
704 ECB : {
705 CBC 26259 : lclContext *ctx = (lclContext *) AH->formatData;
706 GIC 26259 : char b = i; /* Avoid endian problems */
707 ECB :
708 GBC 26259 : if (tarWrite(&b, 1, ctx->FH) != 1)
709 UIC 0 : WRITE_ERROR_EXIT;
710 ECB :
711 CBC 26259 : ctx->filePos += 1;
712 GIC 26259 : return 1;
713 : }
714 :
715 ECB : static int
716 GIC 26259 : _ReadByte(ArchiveHandle *AH)
717 ECB : {
718 GIC 26259 : lclContext *ctx = (lclContext *) AH->formatData;
719 : size_t res;
720 : unsigned char c;
721 ECB :
722 CBC 26259 : res = tarRead(&c, 1, ctx->FH);
723 GIC 26259 : if (res != 1)
724 EUB : /* We already would have exited for errors on reads, must be EOF */
725 LBC 0 : pg_fatal("could not read from input file: end of file");
726 CBC 26259 : ctx->filePos += 1;
727 GIC 26259 : return c;
728 : }
729 :
730 ECB : static void
731 GIC 3135 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
732 ECB : {
733 GIC 3135 : lclContext *ctx = (lclContext *) AH->formatData;
734 ECB :
735 GBC 3135 : if (tarWrite(buf, len, ctx->FH) != len)
736 UIC 0 : WRITE_ERROR_EXIT;
737 ECB :
738 CBC 3135 : ctx->filePos += len;
739 GIC 3135 : }
740 :
741 ECB : static void
742 GIC 3135 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
743 ECB : {
744 GIC 3135 : lclContext *ctx = (lclContext *) AH->formatData;
745 ECB :
746 GIC 3135 : if (tarRead(buf, len, ctx->FH) != len)
747 EUB : /* We already would have exited for errors on reads, must be EOF */
748 UIC 0 : pg_fatal("could not read from input file: end of file");
749 ECB :
750 CBC 3135 : ctx->filePos += len;
751 GIC 3135 : }
752 :
753 ECB : static void
754 GIC 4 : _CloseArchive(ArchiveHandle *AH)
755 ECB : {
756 GIC 4 : lclContext *ctx = (lclContext *) AH->formatData;
757 : TAR_MEMBER *th;
758 : RestoreOptions *ropt;
759 : RestoreOptions *savRopt;
760 : DumpOptions *savDopt;
761 : int savVerbose,
762 : i;
763 ECB :
764 GIC 4 : if (AH->mode == archModeWrite)
765 : {
766 : /*
767 : * Write the Header & TOC to the archive FIRST
768 ECB : */
769 CBC 2 : th = tarOpen(AH, "toc.dat", 'w');
770 2 : ctx->FH = th;
771 2 : WriteHead(AH);
772 2 : WriteToc(AH);
773 GIC 2 : tarClose(AH, th); /* Not needed any more */
774 :
775 : /*
776 : * Now send the data (tables & LOs)
777 ECB : */
778 GIC 2 : WriteDataChunks(AH, NULL);
779 :
780 : /*
781 : * Now this format wants to append a script which does a full restore
782 : * if the files have been extracted.
783 ECB : */
784 GIC 2 : th = tarOpen(AH, "restore.sql", 'w');
785 ECB :
786 GIC 2 : tarPrintf(th, "--\n"
787 : "-- NOTE:\n"
788 : "--\n"
789 : "-- File paths need to be edited. Search for $$PATH$$ and\n"
790 : "-- replace it with the path to the directory containing\n"
791 : "-- the extracted data files.\n"
792 : "--\n");
793 ECB :
794 GIC 2 : AH->CustomOutPtr = _scriptOut;
795 ECB :
796 CBC 2 : ctx->isSpecialScript = 1;
797 GIC 2 : ctx->scriptTH = th;
798 ECB :
799 CBC 2 : ropt = NewRestoreOptions();
800 2 : memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
801 2 : ropt->filename = NULL;
802 2 : ropt->dropSchema = 1;
803 GIC 2 : ropt->superuser = NULL;
804 CBC 2 : ropt->suppressDumpWarnings = true;
805 ECB :
806 GIC 2 : savDopt = AH->public.dopt;
807 CBC 2 : savRopt = AH->public.ropt;
808 :
809 2 : SetArchiveOptions((Archive *) AH, NULL, ropt);
810 ECB :
811 GIC 2 : savVerbose = AH->public.verbose;
812 CBC 2 : AH->public.verbose = 0;
813 :
814 2 : RestoreArchive((Archive *) AH);
815 :
816 2 : SetArchiveOptions((Archive *) AH, savDopt, savRopt);
817 :
818 2 : AH->public.verbose = savVerbose;
819 :
820 2 : tarClose(AH, th);
821 :
822 GIC 2 : ctx->isSpecialScript = 0;
823 :
824 : /*
825 ECB : * EOF marker for tar files is two blocks of NULLs.
826 : */
827 CBC 2050 : for (i = 0; i < TAR_BLOCK_SIZE * 2; i++)
828 EUB : {
829 GIC 2048 : if (fputc(0, ctx->tarFH) == EOF)
830 UIC 0 : WRITE_ERROR_EXIT;
831 : }
832 ECB :
833 : /* Sync the output file if one is defined */
834 GIC 2 : if (AH->dosync && AH->fSpec)
835 1 : (void) fsync_fname(AH->fSpec, false);
836 ECB : }
837 :
838 GIC 4 : AH->FH = NULL;
839 4 : }
840 ECB :
841 : static size_t
842 CBC 1614 : _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
843 : {
844 1614 : lclContext *ctx = (lclContext *) AH->formatData;
845 :
846 GIC 1614 : return tarWrite(buf, len, ctx->scriptTH);
847 : }
848 :
849 : /*
850 : * Large Object support
851 : */
852 :
853 : /*
854 : * Called by the archiver when starting to save all BLOB DATA (not schema).
855 : * This routine should save whatever format-specific information is needed
856 : * to read the LOs back into memory.
857 : *
858 : * It is called just prior to the dumper's DataDumper routine.
859 : *
860 : * Optional, but strongly recommended.
861 : *
862 ECB : */
863 : static void
864 GNC 1 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
865 : {
866 GIC 1 : lclContext *ctx = (lclContext *) AH->formatData;
867 ECB : char fname[K_STD_BUF_SIZE];
868 :
869 CBC 1 : sprintf(fname, "blobs.toc");
870 GNC 1 : ctx->loToc = tarOpen(AH, fname, 'w');
871 GIC 1 : }
872 :
873 : /*
874 : * Called by the archiver when the dumper calls StartLO.
875 : *
876 : * Mandatory.
877 : *
878 : * Must save the passed OID for retrieval at restore-time.
879 ECB : */
880 : static void
881 GNC 2 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
882 ECB : {
883 GIC 2 : lclContext *ctx = (lclContext *) AH->formatData;
884 2 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
885 ECB : char fname[255];
886 EUB :
887 GIC 2 : if (oid == 0)
888 LBC 0 : pg_fatal("invalid OID for large object (%u)", oid);
889 EUB :
890 GNC 2 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
891 LBC 0 : pg_fatal("compression is not supported by tar archive format");
892 :
893 CBC 2 : sprintf(fname, "blob_%u.dat", oid);
894 :
895 GNC 2 : tarPrintf(ctx->loToc, "%u %s\n", oid, fname);
896 ECB :
897 GIC 2 : tctx->TH = tarOpen(AH, fname, 'w');
898 2 : }
899 :
900 : /*
901 : * Called by the archiver when the dumper calls EndLO.
902 : *
903 : * Optional.
904 : *
905 ECB : */
906 : static void
907 GNC 2 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
908 : {
909 CBC 2 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
910 ECB :
911 GIC 2 : tarClose(AH, tctx->TH);
912 2 : }
913 :
914 : /*
915 : * Called by the archiver when finishing saving all BLOB DATA.
916 : *
917 : * Optional.
918 : *
919 ECB : */
920 : static void
921 GNC 1 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
922 : {
923 GIC 1 : lclContext *ctx = (lclContext *) AH->formatData;
924 :
925 : /* Write out a fake zero OID to mark end-of-LOs. */
926 ECB : /* WriteInt(AH, 0); */
927 :
928 GNC 1 : tarClose(AH, ctx->loToc);
929 GIC 1 : }
930 :
931 :
932 :
933 : /*------------
934 : * TAR Support
935 : *------------
936 : */
937 ECB :
938 : static int
939 CBC 4 : tarPrintf(TAR_MEMBER *th, const char *fmt,...)
940 : {
941 4 : int save_errno = errno;
942 : char *p;
943 GIC 4 : size_t len = 128; /* initial assumption about buffer size */
944 : size_t cnt;
945 ECB :
946 : for (;;)
947 GIC 2 : {
948 : va_list args;
949 ECB :
950 : /* Allocate work buffer. */
951 GIC 6 : p = (char *) pg_malloc(len);
952 ECB :
953 : /* Try to format the data. */
954 CBC 6 : errno = save_errno;
955 6 : va_start(args, fmt);
956 GIC 6 : cnt = pvsnprintf(p, len, fmt, args);
957 CBC 6 : va_end(args);
958 ECB :
959 GIC 6 : if (cnt < len)
960 4 : break; /* success */
961 ECB :
962 : /* Release buffer and loop around to try again with larger len. */
963 GIC 2 : free(p);
964 2 : len = cnt;
965 ECB : }
966 :
967 CBC 4 : cnt = tarWrite(p, cnt, th);
968 GIC 4 : free(p);
969 4 : return (int) cnt;
970 : }
971 ECB :
972 : bool
973 GIC 1 : isValidTarHeader(char *header)
974 ECB : {
975 : int sum;
976 CBC 1 : int chk = tarChecksum(header);
977 :
978 1 : sum = read_tar_number(&header[148], 8);
979 EUB :
980 GIC 1 : if (sum != chk)
981 UIC 0 : return false;
982 ECB :
983 : /* POSIX tar format */
984 CBC 1 : if (memcmp(&header[257], "ustar\0", 6) == 0 &&
985 GIC 1 : memcmp(&header[263], "00", 2) == 0)
986 GBC 1 : return true;
987 EUB : /* GNU tar format */
988 UIC 0 : if (memcmp(&header[257], "ustar \0", 8) == 0)
989 UBC 0 : return true;
990 EUB : /* not-quite-POSIX format written by pre-9.3 pg_dump */
991 UIC 0 : if (memcmp(&header[257], "ustar00\0", 8) == 0)
992 UBC 0 : return true;
993 :
994 UIC 0 : return false;
995 : }
996 :
997 ECB : /* Given the member, write the TAR header & copy the file */
998 : static void
999 CBC 34 : _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
1000 ECB : {
1001 GIC 34 : lclContext *ctx = (lclContext *) AH->formatData;
1002 34 : FILE *tmp = th->tmpFH; /* Grab it for convenience */
1003 ECB : char buf[32768];
1004 : size_t cnt;
1005 GIC 34 : pgoff_t len = 0;
1006 : size_t res;
1007 : size_t i,
1008 : pad;
1009 :
1010 : /*
1011 ECB : * Find file len & go back to start.
1012 EUB : */
1013 CBC 34 : if (fseeko(tmp, 0, SEEK_END) != 0)
1014 LBC 0 : pg_fatal("error during file seek: %m");
1015 GBC 34 : th->fileLen = ftello(tmp);
1016 CBC 34 : if (th->fileLen < 0)
1017 UBC 0 : pg_fatal("could not determine seek position in archive file: %m");
1018 GIC 34 : if (fseeko(tmp, 0, SEEK_SET) != 0)
1019 LBC 0 : pg_fatal("error during file seek: %m");
1020 :
1021 CBC 34 : _tarWriteHeader(th);
1022 :
1023 70 : while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1024 EUB : {
1025 CBC 36 : if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1026 UIC 0 : WRITE_ERROR_EXIT;
1027 CBC 36 : len += res;
1028 EUB : }
1029 GIC 34 : if (!feof(tmp))
1030 LBC 0 : READ_ERROR_EXIT(tmp);
1031 EUB :
1032 GIC 34 : if (fclose(tmp) != 0) /* This *should* delete it... */
1033 LBC 0 : pg_fatal("could not close temporary file: %m");
1034 EUB :
1035 GIC 34 : if (len != th->fileLen)
1036 UIC 0 : pg_fatal("actual file length (%lld) does not match expected (%lld)",
1037 ECB : (long long) len, (long long) th->fileLen);
1038 :
1039 GIC 34 : pad = tarPaddingBytesRequired(len);
1040 CBC 14794 : for (i = 0; i < pad; i++)
1041 EUB : {
1042 GIC 14760 : if (fputc('\0', th->tarFH) == EOF)
1043 UIC 0 : WRITE_ERROR_EXIT;
1044 ECB : }
1045 :
1046 GIC 34 : ctx->tarFHpos += len + pad;
1047 34 : }
1048 :
1049 ECB : /* Locate the file in the archive, read header and position to data */
1050 : static TAR_MEMBER *
1051 CBC 32 : _tarPositionTo(ArchiveHandle *AH, const char *filename)
1052 ECB : {
1053 GIC 32 : lclContext *ctx = (lclContext *) AH->formatData;
1054 GNC 32 : TAR_MEMBER *th = pg_malloc0_object(TAR_MEMBER);
1055 : char c;
1056 : char header[TAR_BLOCK_SIZE];
1057 : size_t i,
1058 : len,
1059 : blks;
1060 ECB : int id;
1061 :
1062 GIC 32 : th->AH = AH;
1063 ECB :
1064 : /* Go to end of current file, if any */
1065 CBC 32 : if (ctx->tarFHpos != 0)
1066 : {
1067 GIC 30 : pg_log_debug("moving from position %lld to next member at file position %lld",
1068 ECB : (long long) ctx->tarFHpos, (long long) ctx->tarNextMember);
1069 :
1070 GIC 13761 : while (ctx->tarFHpos < ctx->tarNextMember)
1071 13731 : _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1072 ECB : }
1073 :
1074 GIC 32 : pg_log_debug("now at file position %lld", (long long) ctx->tarFHpos);
1075 :
1076 : /* We are at the start of the file, or at the next member */
1077 ECB :
1078 : /* Get the header */
1079 GBC 32 : if (!_tarGetHeader(AH, th))
1080 EUB : {
1081 UIC 0 : if (filename)
1082 0 : pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1083 : else
1084 : {
1085 : /*
1086 : * We're just scanning the archive for the next file, so return
1087 EUB : * null
1088 : */
1089 UIC 0 : free(th);
1090 0 : return NULL;
1091 : }
1092 ECB : }
1093 :
1094 GBC 32 : while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1095 : {
1096 UBC 0 : pg_log_debug("skipping tar member %s", th->targetFile);
1097 EUB :
1098 UBC 0 : id = atoi(th->targetFile);
1099 UIC 0 : if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1100 0 : pg_fatal("restoring data out of order is not supported in this archive format: "
1101 : "\"%s\" is required, but comes before \"%s\" in the archive file.",
1102 : th->targetFile, filename);
1103 EUB :
1104 : /* Header doesn't match, so read to next header */
1105 UBC 0 : len = th->fileLen;
1106 UIC 0 : len += tarPaddingBytesRequired(th->fileLen);
1107 UBC 0 : blks = len / TAR_BLOCK_SIZE; /* # of tar blocks */
1108 EUB :
1109 UIC 0 : for (i = 0; i < blks; i++)
1110 UBC 0 : _tarReadRaw(AH, &header[0], TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1111 EUB :
1112 UIC 0 : if (!_tarGetHeader(AH, th))
1113 0 : pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1114 ECB : }
1115 :
1116 CBC 64 : ctx->tarNextMember = ctx->tarFHpos + th->fileLen
1117 GIC 32 : + tarPaddingBytesRequired(th->fileLen);
1118 CBC 32 : th->pos = 0;
1119 :
1120 GIC 32 : return th;
1121 : }
1122 :
1123 ECB : /* Read & verify a header */
1124 : static int
1125 CBC 32 : _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
1126 : {
1127 GIC 32 : lclContext *ctx = (lclContext *) AH->formatData;
1128 : char h[TAR_BLOCK_SIZE];
1129 : char tag[100 + 1];
1130 : int sum,
1131 : chk;
1132 ECB : pgoff_t len;
1133 : pgoff_t hPos;
1134 CBC 32 : bool gotBlock = false;
1135 :
1136 GIC 64 : while (!gotBlock)
1137 ECB : {
1138 : /* Save the pos for reporting purposes */
1139 GIC 32 : hPos = ctx->tarFHpos;
1140 ECB :
1141 : /* Read the next tar block, return EOF, exit if short */
1142 GBC 32 : len = _tarReadRaw(AH, h, TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1143 GIC 32 : if (len == 0) /* EOF */
1144 LBC 0 : return 0;
1145 EUB :
1146 GIC 32 : if (len != TAR_BLOCK_SIZE)
1147 UIC 0 : pg_fatal(ngettext("incomplete tar header found (%lu byte)",
1148 : "incomplete tar header found (%lu bytes)",
1149 : len),
1150 : (unsigned long) len);
1151 ECB :
1152 : /* Calc checksum */
1153 GIC 32 : chk = tarChecksum(h);
1154 32 : sum = read_tar_number(&h[148], 8);
1155 :
1156 : /*
1157 : * If the checksum failed, see if it is a null block. If so, silently
1158 ECB : * continue to the next block.
1159 : */
1160 GIC 32 : if (chk == sum)
1161 32 : gotBlock = true;
1162 : else
1163 : {
1164 EUB : int i;
1165 :
1166 UBC 0 : for (i = 0; i < TAR_BLOCK_SIZE; i++)
1167 : {
1168 0 : if (h[i] != 0)
1169 EUB : {
1170 UIC 0 : gotBlock = true;
1171 0 : break;
1172 : }
1173 : }
1174 : }
1175 : }
1176 ECB :
1177 : /* Name field is 100 bytes, might not be null-terminated */
1178 CBC 32 : strlcpy(tag, &h[0], 100 + 1);
1179 :
1180 32 : len = read_tar_number(&h[124], 12);
1181 :
1182 GIC 32 : pg_log_debug("TOC Entry %s at %llu (length %llu, checksum %d)",
1183 ECB : tag, (unsigned long long) hPos, (unsigned long long) len, sum);
1184 EUB :
1185 GIC 32 : if (chk != sum)
1186 UIC 0 : pg_fatal("corrupt tar header found in %s (expected %d, computed %d) file position %llu",
1187 ECB : tag, sum, chk, (unsigned long long) ftello(ctx->tarFH));
1188 :
1189 GIC 32 : th->targetFile = pg_strdup(tag);
1190 CBC 32 : th->fileLen = len;
1191 :
1192 GIC 32 : return 1;
1193 : }
1194 :
1195 ECB :
1196 : static void
1197 GIC 34 : _tarWriteHeader(TAR_MEMBER *th)
1198 : {
1199 ECB : char h[TAR_BLOCK_SIZE];
1200 :
1201 GIC 34 : tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1202 : 0600, 04000, 02000, time(NULL));
1203 ECB :
1204 EUB : /* Now write the completed header. */
1205 CBC 34 : if (fwrite(h, 1, TAR_BLOCK_SIZE, th->tarFH) != TAR_BLOCK_SIZE)
1206 UIC 0 : WRITE_ERROR_EXIT;
1207 GIC 34 : }
|
