Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_backup_directory.c
4 : *
5 : * A directory format dump is a directory, which contains a "toc.dat" file
6 : * for the TOC, and a separate file for each data entry, named "<oid>.dat".
7 : * Large objects are stored in separate files named "blob_<oid>.dat",
8 : * and there's a plain-text TOC file for them called "blobs.toc". If
9 : * compression is used, each data file is individually compressed and the
10 : * ".gz" suffix is added to the filenames. The TOC files are never
11 : * compressed by pg_dump, however they are accepted with the .gz suffix too,
12 : * in case the user has manually compressed them with 'gzip'.
13 : *
14 : * NOTE: This format is identical to the files written in the tar file in
15 : * the 'tar' format, except that we don't write the restore.sql file (TODO),
16 : * and the tar format doesn't support compression. Please keep the formats in
17 : * sync.
18 : *
19 : *
20 : * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
21 : * Portions Copyright (c) 1994, Regents of the University of California
22 : * Portions Copyright (c) 2000, Philip Warner
23 : *
24 : * Rights are granted to use this software in any way so long
25 : * as this notice is not removed.
26 : *
27 : * The author is not responsible for loss or damages that may
28 : * result from its use.
29 : *
30 : * IDENTIFICATION
31 : * src/bin/pg_dump/pg_backup_directory.c
32 : *
33 : *-------------------------------------------------------------------------
34 : */
35 : #include "postgres_fe.h"
36 :
37 : #include <dirent.h>
38 : #include <sys/stat.h>
39 :
40 : #include "common/file_utils.h"
41 : #include "compress_io.h"
42 : #include "parallel.h"
43 : #include "pg_backup_utils.h"
44 :
45 : typedef struct
46 : {
47 : /*
48 : * Our archive location. This is basically what the user specified as his
49 : * backup file but of course here it is a directory.
50 : */
51 : char *directory;
52 :
53 : CompressFileHandle *dataFH; /* currently open data file */
54 : CompressFileHandle *LOsTocFH; /* file handle for blobs.toc */
55 : ParallelState *pstate; /* for parallel backup / restore */
56 : } lclContext;
57 :
58 : typedef struct
59 : {
60 : char *filename; /* filename excluding the directory (basename) */
61 : } lclTocEntry;
62 :
63 : /* prototypes for private functions */
64 : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
65 : static void _StartData(ArchiveHandle *AH, TocEntry *te);
66 : static void _EndData(ArchiveHandle *AH, TocEntry *te);
67 : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
68 : static int _WriteByte(ArchiveHandle *AH, const int i);
69 : static int _ReadByte(ArchiveHandle *AH);
70 : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
71 : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
72 : static void _CloseArchive(ArchiveHandle *AH);
73 : static void _ReopenArchive(ArchiveHandle *AH);
74 : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
75 :
76 : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
77 : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
78 : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
79 :
80 : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
81 : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
82 : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
83 : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
84 : static void _LoadLOs(ArchiveHandle *AH);
85 :
86 : static void _PrepParallelRestore(ArchiveHandle *AH);
87 : static void _Clone(ArchiveHandle *AH);
88 : static void _DeClone(ArchiveHandle *AH);
89 :
90 : static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
91 : static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
92 :
93 : static void setFilePath(ArchiveHandle *AH, char *buf,
94 : const char *relativeFilename);
95 :
96 : /*
97 : * Init routine required by ALL formats. This is a global routine
98 : * and should be declared in pg_backup_archiver.h
99 : *
100 : * Its task is to create any extra archive context (using AH->formatData),
101 : * and to initialize the supported function pointers.
102 : *
103 : * It should also prepare whatever its input source is for reading/writing,
104 : * and in the case of a read mode connection, it should load the Header & TOC.
105 : */
4459 heikki.linnakangas 106 ECB : void
4459 heikki.linnakangas 107 GIC 24 : InitArchiveFmt_Directory(ArchiveHandle *AH)
108 : {
109 : lclContext *ctx;
110 :
4459 heikki.linnakangas 111 ECB : /* Assuming static functions, this can be copied for each format. */
4459 heikki.linnakangas 112 CBC 24 : AH->ArchiveEntryPtr = _ArchiveEntry;
113 24 : AH->StartDataPtr = _StartData;
114 24 : AH->WriteDataPtr = _WriteData;
115 24 : AH->EndDataPtr = _EndData;
116 24 : AH->WriteBytePtr = _WriteByte;
117 24 : AH->ReadBytePtr = _ReadByte;
118 24 : AH->WriteBufPtr = _WriteBuf;
119 24 : AH->ReadBufPtr = _ReadBuf;
120 24 : AH->ClosePtr = _CloseArchive;
3668 andrew 121 24 : AH->ReopenPtr = _ReopenArchive;
4459 heikki.linnakangas 122 24 : AH->PrintTocDataPtr = _PrintTocData;
123 24 : AH->ReadExtraTocPtr = _ReadExtraToc;
124 24 : AH->WriteExtraTocPtr = _WriteExtraToc;
4459 heikki.linnakangas 125 GIC 24 : AH->PrintExtraTocPtr = _PrintExtraToc;
4459 heikki.linnakangas 126 ECB :
125 peter 127 GNC 24 : AH->StartLOsPtr = _StartLOs;
128 24 : AH->StartLOPtr = _StartLO;
129 24 : AH->EndLOPtr = _EndLO;
130 24 : AH->EndLOsPtr = _EndLOs;
4459 heikki.linnakangas 131 ECB :
1668 tgl 132 CBC 24 : AH->PrepParallelRestorePtr = _PrepParallelRestore;
3668 andrew 133 24 : AH->ClonePtr = _Clone;
3668 andrew 134 GIC 24 : AH->DeClonePtr = _DeClone;
3668 andrew 135 ECB :
3668 andrew 136 CBC 24 : AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
3668 andrew 137 GIC 24 : AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
138 :
4459 heikki.linnakangas 139 ECB : /* Set up our private context */
3841 tgl 140 CBC 24 : ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
4459 heikki.linnakangas 141 GIC 24 : AH->formatData = (void *) ctx;
4459 heikki.linnakangas 142 ECB :
4459 heikki.linnakangas 143 CBC 24 : ctx->dataFH = NULL;
125 peter 144 GNC 24 : ctx->LOsTocFH = NULL;
145 :
4459 heikki.linnakangas 146 ECB : /* Initialize LO buffering */
4459 heikki.linnakangas 147 CBC 24 : AH->lo_buf_size = LOBBUFSIZE;
4153 bruce 148 GIC 24 : AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
149 :
150 : /*
151 : * Now open the TOC file
152 : */
4459 heikki.linnakangas 153 ECB :
4459 heikki.linnakangas 154 GBC 24 : if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
366 tgl 155 UIC 0 : pg_fatal("no output directory specified");
4459 heikki.linnakangas 156 ECB :
4459 heikki.linnakangas 157 GIC 24 : ctx->directory = AH->fSpec;
4459 heikki.linnakangas 158 ECB :
4459 heikki.linnakangas 159 GIC 24 : if (AH->mode == archModeWrite)
160 : {
3668 andrew 161 ECB : struct stat st;
3668 andrew 162 GIC 11 : bool is_empty = false;
163 :
3668 andrew 164 ECB : /* we accept an empty existing directory */
3668 andrew 165 GIC 11 : if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
3668 andrew 166 EUB : {
3668 andrew 167 UIC 0 : DIR *dir = opendir(ctx->directory);
3668 andrew 168 EUB :
3668 andrew 169 UIC 0 : if (dir)
170 : {
171 : struct dirent *d;
3668 andrew 172 EUB :
3668 andrew 173 UBC 0 : is_empty = true;
3306 bruce 174 UIC 0 : while (errno = 0, (d = readdir(dir)))
3668 andrew 175 EUB : {
3668 andrew 176 UIC 0 : if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
3668 andrew 177 EUB : {
3668 andrew 178 UBC 0 : is_empty = false;
3668 andrew 179 UIC 0 : break;
180 : }
181 : }
3306 bruce 182 EUB :
3306 bruce 183 UBC 0 : if (errno)
366 tgl 184 UIC 0 : pg_fatal("could not read directory \"%s\": %m",
185 : ctx->directory);
3306 bruce 186 EUB :
3306 bruce 187 UBC 0 : if (closedir(dir))
366 tgl 188 UIC 0 : pg_fatal("could not close directory \"%s\": %m",
189 : ctx->directory);
190 : }
191 : }
3668 andrew 192 ECB :
3668 andrew 193 GBC 11 : if (!is_empty && mkdir(ctx->directory, 0700) < 0)
366 tgl 194 UIC 0 : pg_fatal("could not create directory \"%s\": %m",
195 : ctx->directory);
196 : }
197 : else
198 : { /* Read Mode */
199 : char fname[MAXPGPATH];
200 : CompressFileHandle *tocFH;
4459 heikki.linnakangas 201 ECB :
3668 andrew 202 GIC 13 : setFilePath(AH, fname, "toc.dat");
4459 heikki.linnakangas 203 ECB :
45 tomas.vondra 204 GNC 13 : tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
4459 heikki.linnakangas 205 GBC 13 : if (tocFH == NULL)
366 tgl 206 UIC 0 : pg_fatal("could not open input file \"%s\": %m", fname);
4459 heikki.linnakangas 207 ECB :
4459 heikki.linnakangas 208 GIC 13 : ctx->dataFH = tocFH;
209 :
210 : /*
211 : * The TOC of a directory format dump shares the format code of the
212 : * tar format.
4459 heikki.linnakangas 213 ECB : */
4459 heikki.linnakangas 214 CBC 13 : AH->format = archTar;
215 13 : ReadHead(AH);
216 13 : AH->format = archDirectory;
4459 heikki.linnakangas 217 GIC 13 : ReadToc(AH);
218 :
4459 heikki.linnakangas 219 ECB : /* Nothing else in the file, so close it again... */
17 tomas.vondra 220 GNC 13 : if (!EndCompressFileHandle(tocFH))
366 tgl 221 LBC 0 : pg_fatal("could not close TOC file: %m");
4459 heikki.linnakangas 222 GIC 13 : ctx->dataFH = NULL;
4459 heikki.linnakangas 223 ECB : }
4459 heikki.linnakangas 224 GIC 24 : }
225 :
226 : /*
227 : * Called by the Archiver when the dumper creates a new TOC entry.
228 : *
229 : * We determine the filename for this entry.
230 : */
4459 heikki.linnakangas 231 ECB : static void
4459 heikki.linnakangas 232 GIC 1480 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
233 : {
234 : lclTocEntry *tctx;
235 : char fn[MAXPGPATH];
4459 heikki.linnakangas 236 ECB :
3841 tgl 237 CBC 1480 : tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
1668 238 1480 : if (strcmp(te->desc, "BLOBS") == 0)
239 5 : tctx->filename = pg_strdup("blobs.toc");
1668 tgl 240 GIC 1475 : else if (te->dataDumper)
4459 heikki.linnakangas 241 ECB : {
4459 heikki.linnakangas 242 CBC 145 : snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
4153 bruce 243 GIC 145 : tctx->filename = pg_strdup(fn);
244 : }
4459 heikki.linnakangas 245 ECB : else
4459 heikki.linnakangas 246 GIC 1330 : tctx->filename = NULL;
4459 heikki.linnakangas 247 ECB :
4459 heikki.linnakangas 248 CBC 1480 : te->formatData = (void *) tctx;
4459 heikki.linnakangas 249 GIC 1480 : }
250 :
251 : /*
252 : * Called by the Archiver to save any extra format-related TOC entry
253 : * data.
254 : *
255 : * Use the Archiver routines to write data - they are non-endian, and
256 : * maintain other important file information.
257 : */
4459 heikki.linnakangas 258 ECB : static void
4459 heikki.linnakangas 259 GIC 1480 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
4459 heikki.linnakangas 260 ECB : {
4459 heikki.linnakangas 261 GIC 1480 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
262 :
263 : /*
264 : * A dumpable object has set tctx->filename, any other object has not.
265 : * (see _ArchiveEntry).
4459 heikki.linnakangas 266 ECB : */
4459 heikki.linnakangas 267 CBC 1480 : if (tctx->filename)
4459 heikki.linnakangas 268 GIC 150 : WriteStr(AH, tctx->filename);
4459 heikki.linnakangas 269 ECB : else
4459 heikki.linnakangas 270 CBC 1330 : WriteStr(AH, "");
4459 heikki.linnakangas 271 GIC 1480 : }
272 :
273 : /*
274 : * Called by the Archiver to read any extra format-related TOC data.
275 : *
276 : * Needs to match the order defined in _WriteExtraToc, and should also
277 : * use the Archiver input routines.
278 : */
4459 heikki.linnakangas 279 ECB : static void
4459 heikki.linnakangas 280 GIC 1747 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
4459 heikki.linnakangas 281 ECB : {
4459 heikki.linnakangas 282 GIC 1747 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
4459 heikki.linnakangas 283 ECB :
4459 heikki.linnakangas 284 GIC 1747 : if (tctx == NULL)
4459 heikki.linnakangas 285 ECB : {
3841 tgl 286 CBC 1747 : tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
4459 heikki.linnakangas 287 GIC 1747 : te->formatData = (void *) tctx;
288 : }
4459 heikki.linnakangas 289 ECB :
4459 heikki.linnakangas 290 CBC 1747 : tctx->filename = ReadStr(AH);
4459 heikki.linnakangas 291 GIC 1747 : if (strlen(tctx->filename) == 0)
4459 heikki.linnakangas 292 ECB : {
4459 heikki.linnakangas 293 CBC 1571 : free(tctx->filename);
4459 heikki.linnakangas 294 GIC 1571 : tctx->filename = NULL;
4459 heikki.linnakangas 295 ECB : }
4459 heikki.linnakangas 296 GIC 1747 : }
297 :
298 : /*
299 : * Called by the Archiver when restoring an archive to output a comment
300 : * that includes useful information about the TOC entry.
301 : */
4459 heikki.linnakangas 302 ECB : static void
4459 heikki.linnakangas 303 GIC 1350 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
4459 heikki.linnakangas 304 ECB : {
4459 heikki.linnakangas 305 GIC 1350 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
4459 heikki.linnakangas 306 ECB :
4459 heikki.linnakangas 307 GBC 1350 : if (AH->public.verbose && tctx->filename)
4459 heikki.linnakangas 308 LBC 0 : ahprintf(AH, "-- File: %s\n", tctx->filename);
4459 heikki.linnakangas 309 GIC 1350 : }
310 :
311 : /*
312 : * Called by the archiver when saving TABLE DATA (not schema). This routine
313 : * should save whatever format-specific information is needed to read
314 : * the archive back.
315 : *
316 : * It is called just prior to the dumper's 'DataDumper' routine being called.
317 : *
318 : * We create the data file for writing.
319 : */
4459 heikki.linnakangas 320 ECB : static void
4459 heikki.linnakangas 321 GIC 145 : _StartData(ArchiveHandle *AH, TocEntry *te)
4459 heikki.linnakangas 322 ECB : {
4382 bruce 323 CBC 145 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
4382 bruce 324 GIC 145 : lclContext *ctx = (lclContext *) AH->formatData;
325 : char fname[MAXPGPATH];
4459 heikki.linnakangas 326 ECB :
3668 andrew 327 GIC 145 : setFilePath(AH, fname, tctx->filename);
4459 heikki.linnakangas 328 ECB :
45 tomas.vondra 329 GNC 145 : ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
330 :
17 331 145 : if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
366 tgl 332 UBC 0 : pg_fatal("could not open output file \"%s\": %m", fname);
4459 heikki.linnakangas 333 CBC 145 : }
334 :
335 : /*
336 : * Called by archiver when dumper calls WriteData. This routine is
337 : * called for both LO and table data; it is the responsibility of
338 : * the format to manage each kind of data using StartLO/StartData.
339 : *
340 : * It should only be called from within a DataDumper routine.
341 : *
342 : * We write the data to the open data file.
343 : */
344 : static void
345 24306 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
346 : {
4382 bruce 347 24306 : lclContext *ctx = (lclContext *) AH->formatData;
45 tomas.vondra 348 GNC 24306 : CompressFileHandle *CFH = ctx->dataFH;
4459 heikki.linnakangas 349 ECB :
1024 alvherre 350 GIC 24306 : errno = 0;
17 tomas.vondra 351 GNC 24306 : if (dLen > 0 && !CFH->write_func(data, dLen, CFH))
1024 alvherre 352 ECB : {
353 : /* if write didn't set errno, assume problem is no disk space */
1024 alvherre 354 UIC 0 : if (errno == 0)
1024 alvherre 355 UBC 0 : errno = ENOSPC;
366 tgl 356 0 : pg_fatal("could not write to output file: %s",
357 : CFH->get_error_func(CFH));
358 : }
4459 heikki.linnakangas 359 GIC 24306 : }
4459 heikki.linnakangas 360 ECB :
361 : /*
362 : * Called by the archiver when a dumper's 'DataDumper' routine has
363 : * finished.
364 : *
365 : * We close the data file.
366 : */
367 : static void
4459 heikki.linnakangas 368 GIC 145 : _EndData(ArchiveHandle *AH, TocEntry *te)
4459 heikki.linnakangas 369 ECB : {
4382 bruce 370 GIC 145 : lclContext *ctx = (lclContext *) AH->formatData;
4459 heikki.linnakangas 371 ECB :
372 : /* Close the file */
17 tomas.vondra 373 GNC 145 : if (!EndCompressFileHandle(ctx->dataFH))
366 tgl 374 LBC 0 : pg_fatal("could not close data file: %m");
4459 heikki.linnakangas 375 EUB :
4459 heikki.linnakangas 376 GIC 145 : ctx->dataFH = NULL;
4459 heikki.linnakangas 377 CBC 145 : }
4459 heikki.linnakangas 378 ECB :
379 : /*
380 : * Print data for a given file (can be a LO as well)
381 : */
382 : static void
2643 tgl 383 GIC 156 : _PrintFileData(ArchiveHandle *AH, char *filename)
4459 heikki.linnakangas 384 ECB : {
17 tomas.vondra 385 GNC 156 : size_t cnt = 0;
4459 heikki.linnakangas 386 ECB : char *buf;
387 : size_t buflen;
388 : CompressFileHandle *CFH;
389 :
4459 heikki.linnakangas 390 GIC 156 : if (!filename)
4459 heikki.linnakangas 391 LBC 0 : return;
4459 heikki.linnakangas 392 EUB :
45 tomas.vondra 393 GNC 156 : CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
394 156 : if (!CFH)
366 tgl 395 UBC 0 : pg_fatal("could not open input file \"%s\": %m", filename);
396 :
17 tomas.vondra 397 GNC 156 : buflen = DEFAULT_IO_BUFFER_SIZE;
398 156 : buf = pg_malloc(buflen);
399 :
400 338 : while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
401 : {
4459 heikki.linnakangas 402 CBC 182 : ahwrite(buf, 1, cnt, AH);
403 : }
404 :
405 156 : free(buf);
17 tomas.vondra 406 GNC 156 : if (!EndCompressFileHandle(CFH))
366 tgl 407 UBC 0 : pg_fatal("could not close data file \"%s\": %m", filename);
408 : }
409 :
410 : /*
411 : * Print data for a given TOC entry
412 : */
413 : static void
2643 tgl 414 CBC 151 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
415 : {
4459 heikki.linnakangas 416 151 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
417 :
418 151 : if (!tctx->filename)
4459 heikki.linnakangas 419 UBC 0 : return;
420 :
4459 heikki.linnakangas 421 CBC 151 : if (strcmp(te->desc, "BLOBS") == 0)
125 peter 422 GNC 5 : _LoadLOs(AH);
423 : else
424 : {
425 : char fname[MAXPGPATH];
426 :
3668 andrew 427 CBC 146 : setFilePath(AH, fname, tctx->filename);
2643 tgl 428 146 : _PrintFileData(AH, fname);
429 : }
430 : }
431 :
432 : static void
125 peter 433 GNC 5 : _LoadLOs(ArchiveHandle *AH)
434 : {
435 : Oid oid;
4382 bruce 436 CBC 5 : lclContext *ctx = (lclContext *) AH->formatData;
437 : CompressFileHandle *CFH;
438 : char tocfname[MAXPGPATH];
439 : char line[MAXPGPATH];
440 :
125 peter 441 GNC 5 : StartRestoreLOs(AH);
4459 heikki.linnakangas 442 ECB :
537 dgustafsson 443 GIC 5 : setFilePath(AH, tocfname, "blobs.toc");
4459 heikki.linnakangas 444 ECB :
45 tomas.vondra 445 GNC 5 : CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
4459 heikki.linnakangas 446 ECB :
125 peter 447 GNC 5 : if (ctx->LOsTocFH == NULL)
366 tgl 448 LBC 0 : pg_fatal("could not open large object TOC file \"%s\" for input: %m",
366 tgl 449 EUB : tocfname);
450 :
451 : /* Read the LOs TOC file line-by-line, and process each LO */
45 tomas.vondra 452 GNC 15 : while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
4459 heikki.linnakangas 453 ECB : {
454 : char lofname[MAXPGPATH + 1];
455 : char path[MAXPGPATH];
456 :
457 : /* Can't overflow because line and lofname are the same length */
125 peter 458 GNC 10 : if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, lofname) != 2)
366 tgl 459 LBC 0 : pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
366 tgl 460 EUB : tocfname, line);
461 :
125 peter 462 GNC 10 : StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
463 10 : snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, lofname);
2643 tgl 464 CBC 10 : _PrintFileData(AH, path);
125 peter 465 GNC 10 : EndRestoreLO(AH, oid);
4459 heikki.linnakangas 466 ECB : }
45 tomas.vondra 467 GNC 5 : if (!CFH->eof_func(CFH))
366 tgl 468 LBC 0 : pg_fatal("error reading large object TOC file \"%s\"",
366 tgl 469 EUB : tocfname);
470 :
17 tomas.vondra 471 GNC 5 : if (!EndCompressFileHandle(ctx->LOsTocFH))
366 tgl 472 LBC 0 : pg_fatal("could not close large object TOC file \"%s\": %m",
366 tgl 473 EUB : tocfname);
474 :
125 peter 475 GNC 5 : ctx->LOsTocFH = NULL;
4459 heikki.linnakangas 476 ECB :
125 peter 477 GNC 5 : EndRestoreLOs(AH);
4459 heikki.linnakangas 478 CBC 5 : }
4459 heikki.linnakangas 479 ECB :
480 :
481 : /*
482 : * Write a byte of data to the archive.
483 : * Called by the archiver to do integer & byte output to the archive.
484 : * These routines are only used to read & write the headers & TOC.
485 : */
486 : static int
4459 heikki.linnakangas 487 GIC 138352 : _WriteByte(ArchiveHandle *AH, const int i)
4459 heikki.linnakangas 488 ECB : {
4459 heikki.linnakangas 489 GIC 138352 : unsigned char c = (unsigned char) i;
4459 heikki.linnakangas 490 CBC 138352 : lclContext *ctx = (lclContext *) AH->formatData;
45 tomas.vondra 491 GNC 138352 : CompressFileHandle *CFH = ctx->dataFH;
4459 heikki.linnakangas 492 ECB :
1024 alvherre 493 CBC 138352 : errno = 0;
17 tomas.vondra 494 GNC 138352 : if (!CFH->write_func(&c, 1, CFH))
1024 alvherre 495 ECB : {
496 : /* if write didn't set errno, assume problem is no disk space */
1024 alvherre 497 UIC 0 : if (errno == 0)
498 0 : errno = ENOSPC;
366 tgl 499 UBC 0 : pg_fatal("could not write to output file: %s",
500 : CFH->get_error_func(CFH));
1024 alvherre 501 EUB : }
502 :
4459 heikki.linnakangas 503 GIC 138352 : return 1;
504 : }
4459 heikki.linnakangas 505 ECB :
506 : /*
507 : * Read a byte of data from the archive.
508 : * Called by the archiver to read bytes & integers from the archive.
509 : * These routines are only used to read & write headers & TOC.
510 : * EOF should be treated as a fatal error.
511 : */
512 : static int
4459 heikki.linnakangas 513 GIC 163256 : _ReadByte(ArchiveHandle *AH)
514 : {
4459 heikki.linnakangas 515 CBC 163256 : lclContext *ctx = (lclContext *) AH->formatData;
45 tomas.vondra 516 GNC 163256 : CompressFileHandle *CFH = ctx->dataFH;
517 :
518 163256 : return CFH->getc_func(CFH);
4459 heikki.linnakangas 519 ECB : }
520 :
521 : /*
522 : * Write a buffer of data to the archive.
523 : * Called by the archiver to write a block of bytes to the TOC or a data file.
524 : */
525 : static void
4459 heikki.linnakangas 526 GIC 16609 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
527 : {
528 16609 : lclContext *ctx = (lclContext *) AH->formatData;
45 tomas.vondra 529 GNC 16609 : CompressFileHandle *CFH = ctx->dataFH;
4459 heikki.linnakangas 530 ECB :
1024 alvherre 531 GIC 16609 : errno = 0;
17 tomas.vondra 532 GNC 16609 : if (!CFH->write_func(buf, len, CFH))
1024 alvherre 533 ECB : {
534 : /* if write didn't set errno, assume problem is no disk space */
1024 alvherre 535 LBC 0 : if (errno == 0)
536 0 : errno = ENOSPC;
366 tgl 537 UIC 0 : pg_fatal("could not write to output file: %s",
538 : CFH->get_error_func(CFH));
1024 alvherre 539 EUB : }
4459 heikki.linnakangas 540 GBC 16609 : }
4459 heikki.linnakangas 541 EUB :
542 : /*
543 : * Read a block of bytes from the archive.
4459 heikki.linnakangas 544 ECB : *
545 : * Called by the archiver to read a block of bytes from the archive
546 : */
547 : static void
4459 heikki.linnakangas 548 GIC 19589 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
549 : {
550 19589 : lclContext *ctx = (lclContext *) AH->formatData;
45 tomas.vondra 551 GNC 19589 : CompressFileHandle *CFH = ctx->dataFH;
552 :
3261 bruce 553 ECB : /*
554 : * If there was an I/O error, we already exited in readF(), so here we
3260 555 : * exit on short reads.
3261 556 : */
17 tomas.vondra 557 GNC 19589 : if (!CFH->read_func(buf, len, NULL, CFH))
366 tgl 558 UIC 0 : pg_fatal("could not read from input file: end of file");
4459 heikki.linnakangas 559 GIC 19589 : }
560 :
561 : /*
4459 heikki.linnakangas 562 ECB : * Close the archive.
4459 heikki.linnakangas 563 EUB : *
4459 heikki.linnakangas 564 ECB : * When writing the archive, this is the routine that actually starts
565 : * the process of saving it to files. No data should be written prior
566 : * to this point, since the user could sort the TOC after creating it.
567 : *
568 : * If an archive is to be written, this routine must call:
569 : * WriteHead to save the archive header
570 : * WriteToc to save the TOC entries
571 : * WriteDataChunks to save all data & LOs.
572 : */
573 : static void
2643 tgl 574 GIC 24 : _CloseArchive(ArchiveHandle *AH)
575 : {
4459 heikki.linnakangas 576 24 : lclContext *ctx = (lclContext *) AH->formatData;
577 :
578 24 : if (AH->mode == archModeWrite)
4459 heikki.linnakangas 579 ECB : {
580 : CompressFileHandle *tocFH;
128 michael 581 GNC 11 : pg_compress_specification compression_spec = {0};
3668 andrew 582 ECB : char fname[MAXPGPATH];
583 :
3668 andrew 584 CBC 11 : setFilePath(AH, fname, "toc.dat");
585 :
586 : /* this will actually fork the processes for a parallel backup */
2643 tgl 587 11 : ctx->pstate = ParallelBackupStart(AH);
588 :
589 : /* The TOC is always created uncompressed */
128 michael 590 GNC 11 : compression_spec.algorithm = PG_COMPRESSION_NONE;
45 tomas.vondra 591 11 : tocFH = InitCompressFileHandle(compression_spec);
17 592 11 : if (!tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
366 tgl 593 UIC 0 : pg_fatal("could not open output file \"%s\": %m", fname);
4459 heikki.linnakangas 594 CBC 11 : ctx->dataFH = tocFH;
595 :
596 : /*
4459 heikki.linnakangas 597 ECB : * Write 'tar' in the format field of the toc.dat file. The directory
598 : * is compatible with 'tar', so there's no point having a different
599 : * format code for it.
4459 heikki.linnakangas 600 EUB : */
4459 heikki.linnakangas 601 CBC 11 : AH->format = archTar;
4459 heikki.linnakangas 602 GIC 11 : WriteHead(AH);
603 11 : AH->format = archDirectory;
604 11 : WriteToc(AH);
17 tomas.vondra 605 GNC 11 : if (!EndCompressFileHandle(tocFH))
366 tgl 606 UIC 0 : pg_fatal("could not close TOC file: %m");
2643 tgl 607 GIC 11 : WriteDataChunks(AH, ctx->pstate);
3668 andrew 608 ECB :
3668 andrew 609 CBC 11 : ParallelBackupEnd(AH, ctx->pstate);
2209 andrew 610 ECB :
611 : /*
612 : * In directory mode, there is no need to sync all the entries
2209 andrew 613 EUB : * individually. Just recurse once through all the files generated.
2209 andrew 614 ECB : */
2209 andrew 615 GIC 11 : if (AH->dosync)
1469 peter 616 CBC 5 : fsync_dir_recurse(ctx->directory);
617 : }
4459 heikki.linnakangas 618 GIC 24 : AH->FH = NULL;
619 24 : }
620 :
621 : /*
3668 andrew 622 ECB : * Reopen the archive's file handle.
623 : */
624 : static void
3668 andrew 625 CBC 14 : _ReopenArchive(ArchiveHandle *AH)
3668 andrew 626 ECB : {
627 : /*
628 : * Our TOC is in memory, our data files are opened by each child anyway as
629 : * they are separate. We support reopening the archive by just doing
630 : * nothing.
631 : */
3668 andrew 632 CBC 14 : }
633 :
634 : /*
635 : * LO support
636 : */
637 :
638 : /*
4459 heikki.linnakangas 639 ECB : * Called by the archiver when starting to save all BLOB DATA (not schema).
640 : * It is called just prior to the dumper's DataDumper routine.
641 : *
642 : * We open the large object TOC file here, so that we can append a line to
643 : * it for each LO.
644 : */
645 : static void
125 peter 646 GNC 5 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
647 : {
4382 bruce 648 GIC 5 : lclContext *ctx = (lclContext *) AH->formatData;
128 michael 649 GNC 5 : pg_compress_specification compression_spec = {0};
650 : char fname[MAXPGPATH];
651 :
3668 andrew 652 GIC 5 : setFilePath(AH, fname, "blobs.toc");
653 :
654 : /* The LO TOC file is never compressed */
128 michael 655 GNC 5 : compression_spec.algorithm = PG_COMPRESSION_NONE;
45 tomas.vondra 656 5 : ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
17 657 5 : if (!ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
366 tgl 658 LBC 0 : pg_fatal("could not open output file \"%s\": %m", fname);
4459 heikki.linnakangas 659 GIC 5 : }
660 :
4459 heikki.linnakangas 661 ECB : /*
662 : * Called by the archiver when we're about to start dumping a LO.
663 : *
664 : * We create a file to write the LO to.
665 : */
666 : static void
125 peter 667 GNC 10 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
4459 heikki.linnakangas 668 ECB : {
4382 bruce 669 GIC 10 : lclContext *ctx = (lclContext *) AH->formatData;
670 : char fname[MAXPGPATH];
671 :
4459 heikki.linnakangas 672 10 : snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
673 :
45 tomas.vondra 674 GNC 10 : ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
17 675 10 : if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
366 tgl 676 UIC 0 : pg_fatal("could not open output file \"%s\": %m", fname);
4459 heikki.linnakangas 677 CBC 10 : }
678 :
679 : /*
680 : * Called by the archiver when the dumper is finished writing a LO.
681 : *
682 : * We close the LO file and write an entry to the LO TOC file for it.
4459 heikki.linnakangas 683 ECB : */
4459 heikki.linnakangas 684 EUB : static void
125 peter 685 GNC 10 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
686 : {
4382 bruce 687 GIC 10 : lclContext *ctx = (lclContext *) AH->formatData;
45 tomas.vondra 688 GNC 10 : CompressFileHandle *CFH = ctx->LOsTocFH;
689 : char buf[50];
690 : int len;
691 :
692 : /* Close the BLOB data file itself */
17 693 10 : if (!EndCompressFileHandle(ctx->dataFH))
44 dgustafsson 694 UNC 0 : pg_fatal("could not close LO data file: %m");
4459 heikki.linnakangas 695 GIC 10 : ctx->dataFH = NULL;
4459 heikki.linnakangas 696 ECB :
697 : /* register the LO in blobs.toc */
4459 heikki.linnakangas 698 GIC 10 : len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
17 tomas.vondra 699 GNC 10 : if (!CFH->write_func(buf, len, CFH))
44 dgustafsson 700 UNC 0 : pg_fatal("could not write to LOs TOC file");
4459 heikki.linnakangas 701 GIC 10 : }
4459 heikki.linnakangas 702 ECB :
4459 heikki.linnakangas 703 EUB : /*
4459 heikki.linnakangas 704 ECB : * Called by the archiver when finishing saving all BLOB DATA.
705 : *
706 : * We close the LOs TOC file.
707 : */
708 : static void
125 peter 709 GNC 5 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
4459 heikki.linnakangas 710 ECB : {
4459 heikki.linnakangas 711 GIC 5 : lclContext *ctx = (lclContext *) AH->formatData;
712 :
17 tomas.vondra 713 GNC 5 : if (!EndCompressFileHandle(ctx->LOsTocFH))
44 dgustafsson 714 UNC 0 : pg_fatal("could not close LOs TOC file: %m");
125 peter 715 GNC 5 : ctx->LOsTocFH = NULL;
4459 heikki.linnakangas 716 GIC 5 : }
717 :
3668 andrew 718 ECB : /*
719 : * Gets a relative file name and prepends the output directory, writing the
720 : * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
721 : * big. Can't use a static char[MAXPGPATH] inside the function because we run
722 : * multithreaded on Windows.
3668 andrew 723 EUB : */
3668 andrew 724 ECB : static void
3668 andrew 725 CBC 341 : setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
726 : {
4382 bruce 727 GIC 341 : lclContext *ctx = (lclContext *) AH->formatData;
728 : char *dname;
729 :
4459 heikki.linnakangas 730 341 : dname = ctx->directory;
731 :
732 341 : if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
366 tgl 733 UIC 0 : pg_fatal("file name too long: \"%s\"", dname);
4459 heikki.linnakangas 734 ECB :
4459 heikki.linnakangas 735 GIC 341 : strcpy(buf, dname);
4459 heikki.linnakangas 736 CBC 341 : strcat(buf, "/");
4459 heikki.linnakangas 737 GIC 341 : strcat(buf, relativeFilename);
3668 andrew 738 341 : }
3668 andrew 739 ECB :
740 : /*
1668 tgl 741 : * Prepare for parallel restore.
1668 tgl 742 EUB : *
743 : * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
1668 tgl 744 ECB : * TOC entries' dataLength fields with appropriate values to guide the
745 : * ordering of restore jobs. The source of said data is format-dependent,
746 : * as is the exact meaning of the values.
747 : *
748 : * A format module might also choose to do other setup here.
749 : */
750 : static void
1668 tgl 751 GIC 4 : _PrepParallelRestore(ArchiveHandle *AH)
752 : {
753 : TocEntry *te;
754 :
755 100 : for (te = AH->toc->next; te != AH->toc; te = te->next)
756 : {
757 96 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
758 : char fname[MAXPGPATH];
759 : struct stat st;
1668 tgl 760 ECB :
761 : /*
762 : * A dumpable object has set tctx->filename, any other object has not.
763 : * (see _ArchiveEntry).
764 : */
1668 tgl 765 GIC 96 : if (tctx->filename == NULL)
1668 tgl 766 CBC 80 : continue;
767 :
768 : /* We may ignore items not due to be restored */
1668 tgl 769 GIC 16 : if ((te->reqs & REQ_DATA) == 0)
1668 tgl 770 UIC 0 : continue;
771 :
772 : /*
773 : * Stat the file and, if successful, put its size in dataLength. When
1668 tgl 774 ECB : * using compression, the physical file size might not be a very good
775 : * guide to the amount of work involved in restoring the file, but we
776 : * only need an approximate indicator of that.
777 : */
1668 tgl 778 CBC 16 : setFilePath(AH, fname, tctx->filename);
1668 tgl 779 EUB :
1668 tgl 780 GIC 16 : if (stat(fname, &st) == 0)
1668 tgl 781 UIC 0 : te->dataLength = st.st_size;
45 tomas.vondra 782 GNC 16 : else if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
783 : {
784 16 : if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
785 16 : strlcat(fname, ".gz", sizeof(fname));
45 tomas.vondra 786 UNC 0 : else if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
787 0 : strlcat(fname, ".lz4", sizeof(fname));
4 788 0 : else if (AH->compression_spec.algorithm == PG_COMPRESSION_ZSTD)
789 0 : strlcat(fname, ".zst", sizeof(fname));
790 :
1668 tgl 791 GIC 16 : if (stat(fname, &st) == 0)
1668 tgl 792 CBC 16 : te->dataLength = st.st_size;
793 : }
1668 tgl 794 ECB :
1668 tgl 795 EUB : /*
1668 tgl 796 ECB : * If this is the BLOBS entry, what we stat'd was blobs.toc, which
797 : * most likely is a lot smaller than the actual blob data. We don't
798 : * have a cheap way to estimate how much smaller, but fortunately it
799 : * doesn't matter too much as long as we get the LOs processed
1668 tgl 800 EUB : * reasonably early. Arbitrarily scale up by a factor of 1K.
801 : */
1668 tgl 802 GBC 16 : if (strcmp(te->desc, "BLOBS") == 0)
1668 tgl 803 UBC 0 : te->dataLength *= 1024;
804 : }
1668 tgl 805 CBC 4 : }
1668 tgl 806 ECB :
807 : /*
808 : * Clone format-specific fields during parallel restoration.
809 : */
810 : static void
3668 andrew 811 GIC 28 : _Clone(ArchiveHandle *AH)
812 : {
813 28 : lclContext *ctx = (lclContext *) AH->formatData;
814 :
815 28 : AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
3668 andrew 816 CBC 28 : memcpy(AH->formatData, ctx, sizeof(lclContext));
3668 andrew 817 GBC 28 : ctx = (lclContext *) AH->formatData;
818 :
3668 andrew 819 ECB : /*
820 : * Note: we do not make a local lo_buf because we expect at most one BLOBS
821 : * entry per archive, so no parallelism is possible. Likewise,
822 : * TOC-entry-local state isn't an issue because any one TOC entry is
823 : * touched by just one worker child.
824 : */
825 :
826 : /*
1029 andres 827 : * We also don't copy the ParallelState pointer (pstate), only the leader
828 : * process ever writes to it.
3668 andrew 829 : */
3668 andrew 830 CBC 28 : }
3668 andrew 831 ECB :
832 : static void
3668 andrew 833 GIC 28 : _DeClone(ArchiveHandle *AH)
834 : {
835 28 : lclContext *ctx = (lclContext *) AH->formatData;
836 :
837 28 : free(ctx);
838 28 : }
839 :
840 : /*
841 : * This function is executed in the child of a parallel backup for a
842 : * directory-format archive and dumps the actual data for one TOC entry.
843 : */
2385 tgl 844 ECB : static int
2643 tgl 845 GIC 122 : _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
846 : {
3668 andrew 847 ECB : /*
848 : * This function returns void. We either fail and die horribly or
849 : * succeed... A failure will be detected by the parent when the child dies
850 : * unexpectedly.
851 : */
2643 tgl 852 CBC 122 : WriteDataChunksForTocEntry(AH, te);
853 :
2385 tgl 854 GIC 122 : return 0;
855 : }
856 :
857 : /*
858 : * This function is executed in the child of a parallel restore from a
2385 tgl 859 ECB : * directory-format archive and restores the actual data for one TOC entry.
860 : */
861 : static int
2385 tgl 862 GIC 46 : _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
863 : {
864 46 : return parallel_restore(AH, te);
865 : }
|
