TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * basebackup.c
4 : * code for taking a base backup and streaming it to a standby
5 : *
6 : * Portions Copyright (c) 2010-2023, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/backup/basebackup.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include <sys/stat.h>
16 : #include <unistd.h>
17 : #include <time.h>
18 :
19 : #include "access/xlog_internal.h"
20 : #include "access/xlogbackup.h"
21 : #include "backup/backup_manifest.h"
22 : #include "backup/basebackup.h"
23 : #include "backup/basebackup_sink.h"
24 : #include "backup/basebackup_target.h"
25 : #include "commands/defrem.h"
26 : #include "common/compression.h"
27 : #include "common/file_perm.h"
28 : #include "lib/stringinfo.h"
29 : #include "miscadmin.h"
30 : #include "nodes/pg_list.h"
31 : #include "pgstat.h"
32 : #include "pgtar.h"
33 : #include "port.h"
34 : #include "postmaster/syslogger.h"
35 : #include "replication/walsender.h"
36 : #include "replication/walsender_private.h"
37 : #include "storage/bufpage.h"
38 : #include "storage/checksum.h"
39 : #include "storage/dsm_impl.h"
40 : #include "storage/fd.h"
41 : #include "storage/ipc.h"
42 : #include "storage/reinit.h"
43 : #include "utils/builtins.h"
44 : #include "utils/guc.h"
45 : #include "utils/ps_status.h"
46 : #include "utils/relcache.h"
47 : #include "utils/resowner.h"
48 : #include "utils/timestamp.h"
49 :
50 : /*
51 : * How much data do we want to send in one CopyData message? Note that
52 : * this may also result in reading the underlying files in chunks of this
53 : * size.
54 : *
55 : * NB: The buffer size is required to be a multiple of the system block
56 : * size, so use that value instead if it's bigger than our preference.
57 : */
58 : #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
59 :
60 : typedef struct
61 : {
62 : const char *label;
63 : bool progress;
64 : bool fastcheckpoint;
65 : bool nowait;
66 : bool includewal;
67 : uint32 maxrate;
68 : bool sendtblspcmapfile;
69 : bool send_to_client;
70 : bool use_copytblspc;
71 : BaseBackupTargetHandle *target_handle;
72 : backup_manifest_option manifest;
73 : pg_compress_algorithm compression;
74 : pg_compress_specification compression_specification;
75 : pg_checksum_type manifest_checksum_type;
76 : } basebackup_options;
77 :
78 : static int64 sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly,
79 : struct backup_manifest_info *manifest);
80 : static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
81 : List *tablespaces, bool sendtblspclinks,
82 : backup_manifest_info *manifest, const char *spcoid);
83 : static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
84 : struct stat *statbuf, bool missing_ok, Oid dboid,
85 : backup_manifest_info *manifest, const char *spcoid);
86 : static void sendFileWithContent(bbsink *sink, const char *filename,
87 : const char *content,
88 : backup_manifest_info *manifest);
89 : static int64 _tarWriteHeader(bbsink *sink, const char *filename,
90 : const char *linktarget, struct stat *statbuf,
91 : bool sizeonly);
92 : static void _tarWritePadding(bbsink *sink, int len);
93 : static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
94 : static void perform_base_backup(basebackup_options *opt, bbsink *sink);
95 : static void parse_basebackup_options(List *options, basebackup_options *opt);
96 : static int compareWalFileNames(const ListCell *a, const ListCell *b);
97 : static bool is_checksummed_file(const char *fullpath, const char *filename);
98 : static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
99 : const char *filename, bool partial_read_ok);
100 :
101 : /* Was the backup currently in-progress initiated in recovery mode? */
102 : static bool backup_started_in_recovery = false;
103 :
104 : /* Total number of checksum failures during base backup. */
105 : static long long int total_checksum_failures;
106 :
107 : /* Do not verify checksums. */
108 : static bool noverify_checksums = false;
109 :
110 : /*
111 : * Definition of one element part of an exclusion list, used for paths part
112 : * of checksum validation or base backups. "name" is the name of the file
113 : * or path to check for exclusion. If "match_prefix" is true, any items
114 : * matching the name as prefix are excluded.
115 : */
116 : struct exclude_list_item
117 : {
118 : const char *name;
119 : bool match_prefix;
120 : };
121 :
122 : /*
123 : * The contents of these directories are removed or recreated during server
124 : * start so they are not included in backups. The directories themselves are
125 : * kept and included as empty to preserve access permissions.
126 : *
127 : * Note: this list should be kept in sync with the filter lists in pg_rewind's
128 : * filemap.c.
129 : */
130 : static const char *const excludeDirContents[] =
131 : {
132 : /*
133 : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
134 : * because extensions like pg_stat_statements store data there.
135 : */
136 : PG_STAT_TMP_DIR,
137 :
138 : /*
139 : * It is generally not useful to backup the contents of this directory
140 : * even if the intention is to restore to another primary. See backup.sgml
141 : * for a more detailed description.
142 : */
143 : "pg_replslot",
144 :
145 : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
146 : PG_DYNSHMEM_DIR,
147 :
148 : /* Contents removed on startup, see AsyncShmemInit(). */
149 : "pg_notify",
150 :
151 : /*
152 : * Old contents are loaded for possible debugging but are not required for
153 : * normal operation, see SerialInit().
154 : */
155 : "pg_serial",
156 :
157 : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
158 : "pg_snapshots",
159 :
160 : /* Contents zeroed on startup, see StartupSUBTRANS(). */
161 : "pg_subtrans",
162 :
163 : /* end of list */
164 : NULL
165 : };
166 :
167 : /*
168 : * List of files excluded from backups.
169 : */
170 : static const struct exclude_list_item excludeFiles[] =
171 : {
172 : /* Skip auto conf temporary file. */
173 : {PG_AUTOCONF_FILENAME ".tmp", false},
174 :
175 : /* Skip current log file temporary file */
176 : {LOG_METAINFO_DATAFILE_TMP, false},
177 :
178 : /*
179 : * Skip relation cache because it is rebuilt on startup. This includes
180 : * temporary files.
181 : */
182 : {RELCACHE_INIT_FILENAME, true},
183 :
184 : /*
185 : * backup_label and tablespace_map should not exist in a running cluster
186 : * capable of doing an online backup, but exclude them just in case.
187 : */
188 : {BACKUP_LABEL_FILE, false},
189 : {TABLESPACE_MAP, false},
190 :
191 : /*
192 : * If there's a backup_manifest, it belongs to a backup that was used to
193 : * start this server. It is *not* correct for this backup. Our
194 : * backup_manifest is injected into the backup separately if users want
195 : * it.
196 : */
197 : {"backup_manifest", false},
198 :
199 : {"postmaster.pid", false},
200 : {"postmaster.opts", false},
201 :
202 : /* end of list */
203 : {NULL, false}
204 : };
205 :
206 : /*
207 : * List of files excluded from checksum validation.
208 : *
209 : * Note: this list should be kept in sync with what pg_checksums.c
210 : * includes.
211 : */
212 : static const struct exclude_list_item noChecksumFiles[] = {
213 : {"pg_control", false},
214 : {"pg_filenode.map", false},
215 : {"pg_internal.init", true},
216 : {"PG_VERSION", false},
217 : #ifdef EXEC_BACKEND
218 : {"config_exec_params", true},
219 : #endif
220 : {NULL, false}
221 : };
222 :
223 : /*
224 : * Actually do a base backup for the specified tablespaces.
225 : *
226 : * This is split out mainly to avoid complaints about "variable might be
227 : * clobbered by longjmp" from stupider versions of gcc.
228 : */
229 : static void
230 GIC 126 : perform_base_backup(basebackup_options *opt, bbsink *sink)
231 : {
232 ECB : bbsink_state state;
233 : XLogRecPtr endptr;
234 : TimeLineID endtli;
235 : backup_manifest_info manifest;
236 : BackupState *backup_state;
237 : StringInfo tablespace_map;
238 :
239 : /* Initial backup state, insofar as we know it now. */
240 GIC 126 : state.tablespaces = NIL;
241 126 : state.tablespace_num = 0;
242 CBC 126 : state.bytes_done = 0;
243 126 : state.bytes_total = 0;
244 126 : state.bytes_total_is_valid = false;
245 ECB :
246 : /* we're going to use a BufFile, so we need a ResourceOwner */
247 GIC 126 : Assert(CurrentResourceOwner == NULL);
248 126 : CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
249 ECB :
250 CBC 126 : backup_started_in_recovery = RecoveryInProgress();
251 :
252 126 : InitializeBackupManifest(&manifest, opt->manifest,
253 : opt->manifest_checksum_type);
254 :
255 126 : total_checksum_failures = 0;
256 :
257 : /* Allocate backup related varilables. */
258 GNC 126 : backup_state = (BackupState *) palloc0(sizeof(BackupState));
259 126 : tablespace_map = makeStringInfo();
260 :
261 GIC 126 : basebackup_progress_wait_checkpoint();
262 GNC 126 : do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
263 : backup_state, tablespace_map);
264 :
265 126 : state.startptr = backup_state->startpoint;
266 126 : state.starttli = backup_state->starttli;
267 ECB :
268 : /*
269 : * Once do_pg_backup_start has been called, ensure that any failure causes
270 : * us to abort the backup so we don't "leak" a backup counter. For this
271 : * reason, *all* functionality between do_pg_backup_start() and the end of
272 : * do_pg_backup_stop() should be inside the error cleanup block!
273 : */
274 :
275 GIC 126 : PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
276 : {
277 : ListCell *lc;
278 : tablespaceinfo *newti;
279 :
280 ECB : /* Add a node for the base directory at the end */
281 GNC 126 : newti = palloc0(sizeof(tablespaceinfo));
282 126 : newti->size = -1;
283 126 : state.tablespaces = lappend(state.tablespaces, newti);
284 :
285 : /*
286 ECB : * Calculate the total backup size by summing up the size of each
287 : * tablespace
288 : */
289 GIC 126 : if (opt->progress)
290 : {
291 126 : basebackup_progress_estimate_backup_size();
292 :
293 268 : foreach(lc, state.tablespaces)
294 ECB : {
295 GIC 142 : tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
296 ECB :
297 GIC 142 : if (tmp->path == NULL)
298 CBC 126 : tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
299 : true, NULL, NULL);
300 ECB : else
301 GIC 16 : tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
302 ECB : NULL);
303 CBC 142 : state.bytes_total += tmp->size;
304 : }
305 GIC 126 : state.bytes_total_is_valid = true;
306 ECB : }
307 :
308 : /* notify basebackup sink about start of backup */
309 GIC 126 : bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
310 ECB :
311 : /* Send off our tablespaces one by one */
312 GIC 262 : foreach(lc, state.tablespaces)
313 : {
314 CBC 142 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
315 :
316 GIC 142 : if (ti->path == NULL)
317 ECB : {
318 : struct stat statbuf;
319 CBC 126 : bool sendtblspclinks = true;
320 : char *backup_label;
321 :
322 126 : bbsink_begin_archive(sink, "base.tar");
323 :
324 : /* In the main tar, include the backup_label first... */
325 GNC 126 : backup_label = build_backup_content(backup_state, false);
326 126 : sendFileWithContent(sink, BACKUP_LABEL_FILE,
327 : backup_label, &manifest);
328 126 : pfree(backup_label);
329 :
330 ECB : /* Then the tablespace_map file, if required... */
331 GIC 126 : if (opt->sendtblspcmapfile)
332 : {
333 GNC 24 : sendFileWithContent(sink, TABLESPACE_MAP,
334 24 : tablespace_map->data, &manifest);
335 GIC 24 : sendtblspclinks = false;
336 ECB : }
337 :
338 : /* Then the bulk of the files... */
339 CBC 126 : sendDir(sink, ".", 1, false, state.tablespaces,
340 : sendtblspclinks, &manifest, NULL);
341 ECB :
342 : /* ... and pg_control after everything else. */
343 CBC 120 : if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
344 UIC 0 : ereport(ERROR,
345 : (errcode_for_file_access(),
346 : errmsg("could not stat file \"%s\": %m",
347 ECB : XLOG_CONTROL_FILE)));
348 GIC 120 : sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
349 : false, InvalidOid, &manifest, NULL);
350 : }
351 ECB : else
352 EUB : {
353 GIC 16 : char *archive_name = psprintf("%s.tar", ti->oid);
354 :
355 16 : bbsink_begin_archive(sink, archive_name);
356 ECB :
357 GIC 16 : sendTablespace(sink, ti->path, ti->oid, false, &manifest);
358 : }
359 :
360 : /*
361 ECB : * If we're including WAL, and this is the main data directory we
362 : * don't treat this as the end of the tablespace. Instead, we will
363 : * include the xlog files below and stop afterwards. This is safe
364 : * since the main data directory is always sent *last*.
365 : */
366 GIC 136 : if (opt->includewal && ti->path == NULL)
367 : {
368 20 : Assert(lnext(state.tablespaces, lc) == NULL);
369 : }
370 : else
371 : {
372 : /* Properly terminate the tarfile. */
373 : StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ,
374 ECB : "BLCKSZ too small for 2 tar blocks");
375 GIC 116 : memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
376 CBC 116 : bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
377 :
378 : /* OK, that's the end of the archive. */
379 GIC 116 : bbsink_end_archive(sink);
380 : }
381 : }
382 :
383 CBC 120 : basebackup_progress_wait_wal_archive(&state);
384 GNC 120 : do_pg_backup_stop(backup_state, !opt->nowait);
385 :
386 120 : endptr = backup_state->stoppoint;
387 120 : endtli = backup_state->stoptli;
388 :
389 : /* Deallocate backup-related variables. */
390 120 : pfree(tablespace_map->data);
391 120 : pfree(tablespace_map);
392 120 : pfree(backup_state);
393 : }
394 GIC 121 : PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
395 ECB :
396 :
397 GIC 120 : if (opt->includewal)
398 : {
399 ECB : /*
400 : * We've left the last tar file "open", so we can now append the
401 : * required WAL files to it.
402 : */
403 : char pathbuf[MAXPGPATH];
404 : XLogSegNo segno;
405 : XLogSegNo startsegno;
406 : XLogSegNo endsegno;
407 : struct stat statbuf;
408 CBC 20 : List *historyFileList = NIL;
409 GIC 20 : List *walFileList = NIL;
410 ECB : char firstoff[MAXFNAMELEN];
411 : char lastoff[MAXFNAMELEN];
412 : DIR *dir;
413 : struct dirent *de;
414 : ListCell *lc;
415 : TimeLineID tli;
416 :
417 GIC 20 : basebackup_progress_transfer_wal();
418 :
419 : /*
420 : * I'd rather not worry about timelines here, so scan pg_wal and
421 : * include all WAL files in the range between 'startptr' and 'endptr',
422 : * regardless of the timeline the file is stamped with. If there are
423 : * some spurious WAL files belonging to timelines that don't belong in
424 ECB : * this server's history, they will be included too. Normally there
425 : * shouldn't be such files, but if there are, there's little harm in
426 : * including them.
427 : */
428 GIC 20 : XLByteToSeg(state.startptr, startsegno, wal_segment_size);
429 20 : XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
430 20 : XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
431 20 : XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
432 :
433 CBC 20 : dir = AllocateDir("pg_wal");
434 GIC 123 : while ((de = ReadDir(dir, "pg_wal")) != NULL)
435 : {
436 : /* Does it look like a WAL segment, and is it in the range? */
437 103 : if (IsXLogFileName(de->d_name) &&
438 43 : strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
439 43 : strcmp(de->d_name + 8, lastoff + 8) <= 0)
440 : {
441 20 : walFileList = lappend(walFileList, pstrdup(de->d_name));
442 : }
443 : /* Does it look like a timeline history file? */
444 CBC 83 : else if (IsTLHistoryFileName(de->d_name))
445 ECB : {
446 LBC 0 : historyFileList = lappend(historyFileList, pstrdup(de->d_name));
447 ECB : }
448 : }
449 CBC 20 : FreeDir(dir);
450 ECB :
451 : /*
452 : * Before we go any further, check that none of the WAL segments we
453 : * need were removed.
454 : */
455 CBC 20 : CheckXLogRemoved(startsegno, state.starttli);
456 :
457 ECB : /*
458 : * Sort the WAL filenames. We want to send the files in order from
459 : * oldest to newest, to reduce the chance that a file is recycled
460 : * before we get a chance to send it over.
461 : */
462 GBC 20 : list_sort(walFileList, compareWalFileNames);
463 :
464 : /*
465 ECB : * There must be at least one xlog file in the pg_wal directory, since
466 : * we are doing backup-including-xlog.
467 : */
468 GIC 20 : if (walFileList == NIL)
469 UIC 0 : ereport(ERROR,
470 : (errmsg("could not find any WAL files")));
471 ECB :
472 : /*
473 : * Sanity check: the first and last segment should cover startptr and
474 : * endptr, with no gaps in between.
475 : */
476 GIC 20 : XLogFromFileName((char *) linitial(walFileList),
477 : &tli, &segno, wal_segment_size);
478 CBC 20 : if (segno != startsegno)
479 : {
480 : char startfname[MAXFNAMELEN];
481 :
482 UIC 0 : XLogFileName(startfname, state.starttli, startsegno,
483 : wal_segment_size);
484 LBC 0 : ereport(ERROR,
485 EUB : (errmsg("could not find WAL file \"%s\"", startfname)));
486 : }
487 GIC 40 : foreach(lc, walFileList)
488 : {
489 20 : char *walFileName = (char *) lfirst(lc);
490 20 : XLogSegNo currsegno = segno;
491 20 : XLogSegNo nextsegno = segno + 1;
492 ECB :
493 GIC 20 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
494 CBC 20 : if (!(nextsegno == segno || currsegno == segno))
495 : {
496 : char nextfname[MAXFNAMELEN];
497 :
498 UBC 0 : XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
499 UIC 0 : ereport(ERROR,
500 EUB : (errmsg("could not find WAL file \"%s\"", nextfname)));
501 : }
502 : }
503 CBC 20 : if (segno != endsegno)
504 : {
505 ECB : char endfname[MAXFNAMELEN];
506 :
507 LBC 0 : XLogFileName(endfname, endtli, endsegno, wal_segment_size);
508 UIC 0 : ereport(ERROR,
509 ECB : (errmsg("could not find WAL file \"%s\"", endfname)));
510 : }
511 :
512 : /* Ok, we have everything we need. Send the WAL files. */
513 GIC 40 : foreach(lc, walFileList)
514 EUB : {
515 GBC 20 : char *walFileName = (char *) lfirst(lc);
516 : int fd;
517 : size_t cnt;
518 GIC 20 : pgoff_t len = 0;
519 ECB :
520 GIC 20 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
521 20 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
522 :
523 GBC 20 : fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
524 20 : if (fd < 0)
525 : {
526 UIC 0 : int save_errno = errno;
527 :
528 : /*
529 ECB : * Most likely reason for this is that the file was already
530 : * removed by a checkpoint, so check for that to get a better
531 : * error message.
532 : */
533 UIC 0 : CheckXLogRemoved(segno, tli);
534 ECB :
535 UIC 0 : errno = save_errno;
536 LBC 0 : ereport(ERROR,
537 ECB : (errcode_for_file_access(),
538 : errmsg("could not open file \"%s\": %m", pathbuf)));
539 : }
540 :
541 GIC 20 : if (fstat(fd, &statbuf) != 0)
542 UBC 0 : ereport(ERROR,
543 : (errcode_for_file_access(),
544 : errmsg("could not stat file \"%s\": %m",
545 : pathbuf)));
546 GIC 20 : if (statbuf.st_size != wal_segment_size)
547 : {
548 UIC 0 : CheckXLogRemoved(segno, tli);
549 UBC 0 : ereport(ERROR,
550 : (errcode_for_file_access(),
551 EUB : errmsg("unexpected WAL file size \"%s\"", walFileName)));
552 : }
553 :
554 : /* send the WAL file itself */
555 GIC 20 : _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
556 :
557 CBC 20 : while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
558 GBC 10240 : Min(sink->bbs_buffer_length,
559 : wal_segment_size - len),
560 GIC 10240 : len, pathbuf, true)) > 0)
561 : {
562 CBC 10240 : CheckXLogRemoved(segno, tli);
563 GIC 10240 : bbsink_archive_contents(sink, cnt);
564 EUB :
565 GBC 10240 : len += cnt;
566 :
567 GIC 10240 : if (len == wal_segment_size)
568 20 : break;
569 : }
570 :
571 CBC 20 : if (len != wal_segment_size)
572 : {
573 LBC 0 : CheckXLogRemoved(segno, tli);
574 0 : ereport(ERROR,
575 : (errcode_for_file_access(),
576 ECB : errmsg("unexpected WAL file size \"%s\"", walFileName)));
577 : }
578 :
579 : /*
580 : * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
581 : * for padding.
582 : */
583 CBC 20 : Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
584 ECB :
585 GIC 20 : CloseTransientFile(fd);
586 :
587 ECB : /*
588 : * Mark file as archived, otherwise files can get archived again
589 EUB : * after promotion of a new node. This is in line with
590 : * walreceiver.c always doing an XLogArchiveForceDone() after a
591 : * complete segment.
592 : */
593 GIC 20 : StatusFilePath(pathbuf, walFileName, ".done");
594 20 : sendFileWithContent(sink, pathbuf, "", &manifest);
595 : }
596 :
597 : /*
598 : * Send timeline history files too. Only the latest timeline history
599 ECB : * file is required for recovery, and even that only if there happens
600 : * to be a timeline switch in the first WAL segment that contains the
601 : * checkpoint record, or if we're taking a base backup from a standby
602 : * server and the target timeline changes while the backup is taken.
603 : * But they are small and highly useful for debugging purposes, so
604 : * better include them all, always.
605 : */
606 GIC 20 : foreach(lc, historyFileList)
607 : {
608 UIC 0 : char *fname = lfirst(lc);
609 ECB :
610 LBC 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
611 :
612 UIC 0 : if (lstat(pathbuf, &statbuf) != 0)
613 0 : ereport(ERROR,
614 : (errcode_for_file_access(),
615 : errmsg("could not stat file \"%s\": %m", pathbuf)));
616 :
617 0 : sendFile(sink, pathbuf, pathbuf, &statbuf, false, InvalidOid,
618 : &manifest, NULL);
619 :
620 : /* unconditionally mark file as archived */
621 0 : StatusFilePath(pathbuf, fname, ".done");
622 LBC 0 : sendFileWithContent(sink, pathbuf, "", &manifest);
623 : }
624 EUB :
625 : /* Properly terminate the tar file. */
626 : StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
627 : "BLCKSZ too small for 2 tar blocks");
628 GBC 20 : memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
629 20 : bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
630 :
631 : /* OK, that's the end of the archive. */
632 GIC 20 : bbsink_end_archive(sink);
633 EUB : }
634 :
635 GIC 120 : AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
636 : endptr, endtli);
637 EUB :
638 GBC 120 : SendBackupManifest(&manifest, sink);
639 :
640 GIC 120 : bbsink_end_backup(sink, endptr, endtli);
641 :
642 120 : if (total_checksum_failures)
643 : {
644 CBC 3 : if (total_checksum_failures > 1)
645 2 : ereport(WARNING,
646 : (errmsg_plural("%lld total checksum verification failure",
647 : "%lld total checksum verification failures",
648 ECB : total_checksum_failures,
649 : total_checksum_failures)));
650 :
651 CBC 3 : ereport(ERROR,
652 : (errcode(ERRCODE_DATA_CORRUPTED),
653 : errmsg("checksum verification failure during base backup")));
654 ECB : }
655 :
656 : /*
657 : * Make sure to free the manifest before the resource owners as manifests
658 : * use cryptohash contexts that may depend on resource owners (like
659 : * OpenSSL).
660 : */
661 CBC 117 : FreeBackupManifest(&manifest);
662 :
663 : /* clean up the resource owner we created */
664 GIC 117 : WalSndResourceCleanup(true);
665 :
666 117 : basebackup_progress_done();
667 CBC 117 : }
668 :
669 : /*
670 : * list_sort comparison function, to compare log/seg portion of WAL segment
671 : * filenames, ignoring the timeline portion.
672 : */
673 : static int
674 UIC 0 : compareWalFileNames(const ListCell *a, const ListCell *b)
675 : {
676 0 : char *fna = (char *) lfirst(a);
677 LBC 0 : char *fnb = (char *) lfirst(b);
678 :
679 UIC 0 : return strcmp(fna + 8, fnb + 8);
680 ECB : }
681 :
682 : /*
683 : * Parse the base backup options passed down by the parser
684 : */
685 : static void
686 GIC 142 : parse_basebackup_options(List *options, basebackup_options *opt)
687 : {
688 : ListCell *lopt;
689 142 : bool o_label = false;
690 GBC 142 : bool o_progress = false;
691 GIC 142 : bool o_checkpoint = false;
692 GBC 142 : bool o_nowait = false;
693 142 : bool o_wal = false;
694 GIC 142 : bool o_maxrate = false;
695 GBC 142 : bool o_tablespace_map = false;
696 GIC 142 : bool o_noverify_checksums = false;
697 142 : bool o_manifest = false;
698 142 : bool o_manifest_checksums = false;
699 142 : bool o_target = false;
700 142 : bool o_target_detail = false;
701 142 : char *target_str = NULL;
702 CBC 142 : char *target_detail_str = NULL;
703 GIC 142 : bool o_compression = false;
704 142 : bool o_compression_detail = false;
705 CBC 142 : char *compression_detail_str = NULL;
706 ECB :
707 CBC 1562 : MemSet(opt, 0, sizeof(*opt));
708 142 : opt->manifest = MANIFEST_OPTION_NO;
709 142 : opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
710 142 : opt->compression = PG_COMPRESSION_NONE;
711 142 : opt->compression_specification.algorithm = PG_COMPRESSION_NONE;
712 ECB :
713 CBC 1087 : foreach(lopt, options)
714 ECB : {
715 CBC 947 : DefElem *defel = (DefElem *) lfirst(lopt);
716 ECB :
717 CBC 947 : if (strcmp(defel->defname, "label") == 0)
718 ECB : {
719 CBC 142 : if (o_label)
720 LBC 0 : ereport(ERROR,
721 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
722 : errmsg("duplicate option \"%s\"", defel->defname)));
723 CBC 142 : opt->label = defGetString(defel);
724 142 : o_label = true;
725 ECB : }
726 CBC 805 : else if (strcmp(defel->defname, "progress") == 0)
727 ECB : {
728 GIC 142 : if (o_progress)
729 LBC 0 : ereport(ERROR,
730 : (errcode(ERRCODE_SYNTAX_ERROR),
731 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
732 GIC 142 : opt->progress = defGetBoolean(defel);
733 CBC 142 : o_progress = true;
734 : }
735 663 : else if (strcmp(defel->defname, "checkpoint") == 0)
736 EUB : {
737 GIC 132 : char *optval = defGetString(defel);
738 :
739 CBC 132 : if (o_checkpoint)
740 LBC 0 : ereport(ERROR,
741 : (errcode(ERRCODE_SYNTAX_ERROR),
742 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
743 GIC 132 : if (pg_strcasecmp(optval, "fast") == 0)
744 CBC 132 : opt->fastcheckpoint = true;
745 UBC 0 : else if (pg_strcasecmp(optval, "spread") == 0)
746 UIC 0 : opt->fastcheckpoint = false;
747 : else
748 LBC 0 : ereport(ERROR,
749 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
750 : errmsg("unrecognized checkpoint type: \"%s\"",
751 : optval)));
752 GIC 132 : o_checkpoint = true;
753 ECB : }
754 GIC 531 : else if (strcmp(defel->defname, "wait") == 0)
755 ECB : {
756 GBC 135 : if (o_nowait)
757 UIC 0 : ereport(ERROR,
758 : (errcode(ERRCODE_SYNTAX_ERROR),
759 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
760 CBC 135 : opt->nowait = !defGetBoolean(defel);
761 GBC 135 : o_nowait = true;
762 EUB : }
763 GIC 396 : else if (strcmp(defel->defname, "wal") == 0)
764 EUB : {
765 GIC 24 : if (o_wal)
766 UIC 0 : ereport(ERROR,
767 : (errcode(ERRCODE_SYNTAX_ERROR),
768 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
769 GIC 24 : opt->includewal = defGetBoolean(defel);
770 CBC 24 : o_wal = true;
771 : }
772 372 : else if (strcmp(defel->defname, "max_rate") == 0)
773 EUB : {
774 : int64 maxrate;
775 :
776 CBC 1 : if (o_maxrate)
777 LBC 0 : ereport(ERROR,
778 : (errcode(ERRCODE_SYNTAX_ERROR),
779 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
780 :
781 CBC 1 : maxrate = defGetInt64(defel);
782 GBC 1 : if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
783 UIC 0 : ereport(ERROR,
784 : (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
785 ECB : errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
786 : (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
787 :
788 CBC 1 : opt->maxrate = (uint32) maxrate;
789 GIC 1 : o_maxrate = true;
790 : }
791 371 : else if (strcmp(defel->defname, "tablespace_map") == 0)
792 ECB : {
793 GBC 29 : if (o_tablespace_map)
794 UIC 0 : ereport(ERROR,
795 : (errcode(ERRCODE_SYNTAX_ERROR),
796 : errmsg("duplicate option \"%s\"", defel->defname)));
797 CBC 29 : opt->sendtblspcmapfile = defGetBoolean(defel);
798 29 : o_tablespace_map = true;
799 EUB : }
800 GIC 342 : else if (strcmp(defel->defname, "verify_checksums") == 0)
801 : {
802 1 : if (o_noverify_checksums)
803 UIC 0 : ereport(ERROR,
804 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
805 : errmsg("duplicate option \"%s\"", defel->defname)));
806 GIC 1 : noverify_checksums = !defGetBoolean(defel);
807 CBC 1 : o_noverify_checksums = true;
808 : }
809 341 : else if (strcmp(defel->defname, "manifest") == 0)
810 EUB : {
811 GIC 141 : char *optval = defGetString(defel);
812 : bool manifest_bool;
813 ECB :
814 CBC 141 : if (o_manifest)
815 UIC 0 : ereport(ERROR,
816 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
817 : errmsg("duplicate option \"%s\"", defel->defname)));
818 CBC 141 : if (parse_bool(optval, &manifest_bool))
819 EUB : {
820 GIC 140 : if (manifest_bool)
821 140 : opt->manifest = MANIFEST_OPTION_YES;
822 ECB : else
823 LBC 0 : opt->manifest = MANIFEST_OPTION_NO;
824 : }
825 CBC 1 : else if (pg_strcasecmp(optval, "force-encode") == 0)
826 GIC 1 : opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
827 ECB : else
828 UIC 0 : ereport(ERROR,
829 : (errcode(ERRCODE_SYNTAX_ERROR),
830 ECB : errmsg("unrecognized manifest option: \"%s\"",
831 EUB : optval)));
832 GIC 141 : o_manifest = true;
833 : }
834 CBC 200 : else if (strcmp(defel->defname, "manifest_checksums") == 0)
835 : {
836 7 : char *optval = defGetString(defel);
837 ECB :
838 GIC 7 : if (o_manifest_checksums)
839 UBC 0 : ereport(ERROR,
840 : (errcode(ERRCODE_SYNTAX_ERROR),
841 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
842 CBC 7 : if (!pg_checksum_parse_type(optval,
843 : &opt->manifest_checksum_type))
844 GBC 1 : ereport(ERROR,
845 : (errcode(ERRCODE_SYNTAX_ERROR),
846 : errmsg("unrecognized checksum algorithm: \"%s\"",
847 : optval)));
848 CBC 6 : o_manifest_checksums = true;
849 : }
850 193 : else if (strcmp(defel->defname, "target") == 0)
851 : {
852 141 : if (o_target)
853 UIC 0 : ereport(ERROR,
854 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
855 EUB : errmsg("duplicate option \"%s\"", defel->defname)));
856 GIC 141 : target_str = defGetString(defel);
857 141 : o_target = true;
858 ECB : }
859 GIC 52 : else if (strcmp(defel->defname, "target_detail") == 0)
860 ECB : {
861 GIC 9 : char *optval = defGetString(defel);
862 :
863 9 : if (o_target_detail)
864 LBC 0 : ereport(ERROR,
865 : (errcode(ERRCODE_SYNTAX_ERROR),
866 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
867 GIC 9 : target_detail_str = optval;
868 CBC 9 : o_target_detail = true;
869 EUB : }
870 GIC 43 : else if (strcmp(defel->defname, "compression") == 0)
871 : {
872 CBC 29 : char *optval = defGetString(defel);
873 ECB :
874 GIC 29 : if (o_compression)
875 LBC 0 : ereport(ERROR,
876 : (errcode(ERRCODE_SYNTAX_ERROR),
877 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
878 GIC 29 : if (!parse_compress_algorithm(optval, &opt->compression))
879 CBC 1 : ereport(ERROR,
880 EUB : (errcode(ERRCODE_SYNTAX_ERROR),
881 : errmsg("unrecognized compression algorithm: \"%s\"",
882 : optval)));
883 CBC 28 : o_compression = true;
884 ECB : }
885 GIC 14 : else if (strcmp(defel->defname, "compression_detail") == 0)
886 ECB : {
887 GIC 14 : if (o_compression_detail)
888 LBC 0 : ereport(ERROR,
889 : (errcode(ERRCODE_SYNTAX_ERROR),
890 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
891 GBC 14 : compression_detail_str = defGetString(defel);
892 GIC 14 : o_compression_detail = true;
893 : }
894 ECB : else
895 LBC 0 : ereport(ERROR,
896 : (errcode(ERRCODE_SYNTAX_ERROR),
897 : errmsg("unrecognized base backup option: \"%s\"",
898 : defel->defname)));
899 ECB : }
900 :
901 CBC 140 : if (opt->label == NULL)
902 UIC 0 : opt->label = "base backup";
903 CBC 140 : if (opt->manifest == MANIFEST_OPTION_NO)
904 EUB : {
905 GIC 1 : if (o_manifest_checksums)
906 UIC 0 : ereport(ERROR,
907 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
908 : errmsg("manifest checksums require a backup manifest")));
909 GIC 1 : opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
910 : }
911 EUB :
912 GIC 140 : if (target_str == NULL)
913 : {
914 UIC 0 : if (target_detail_str != NULL)
915 0 : ereport(ERROR,
916 : (errcode(ERRCODE_SYNTAX_ERROR),
917 ECB : errmsg("target detail cannot be used without target")));
918 UBC 0 : opt->use_copytblspc = true;
919 LBC 0 : opt->send_to_client = true;
920 : }
921 CBC 140 : else if (strcmp(target_str, "client") == 0)
922 EUB : {
923 GIC 125 : if (target_detail_str != NULL)
924 UIC 0 : ereport(ERROR,
925 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
926 : errmsg("target \"%s\" does not accept a target detail",
927 : target_str)));
928 CBC 125 : opt->send_to_client = true;
929 : }
930 EUB : else
931 GBC 13 : opt->target_handle =
932 GIC 15 : BaseBackupGetTargetHandle(target_str, target_detail_str);
933 :
934 GBC 138 : if (o_compression_detail && !o_compression)
935 UBC 0 : ereport(ERROR,
936 : (errcode(ERRCODE_SYNTAX_ERROR),
937 ECB : errmsg("compression detail cannot be specified unless compression is enabled")));
938 :
939 CBC 138 : if (o_compression)
940 EUB : {
941 : char *error_detail;
942 :
943 GIC 26 : parse_compress_specification(opt->compression, compression_detail_str,
944 ECB : &opt->compression_specification);
945 : error_detail =
946 GIC 26 : validate_compress_specification(&opt->compression_specification);
947 CBC 26 : if (error_detail != NULL)
948 9 : ereport(ERROR,
949 : errcode(ERRCODE_SYNTAX_ERROR),
950 ECB : errmsg("invalid compression specification: %s",
951 EUB : error_detail));
952 : }
953 GIC 129 : }
954 :
955 ECB :
956 : /*
957 : * SendBaseBackup() - send a complete base backup.
958 : *
959 : * The function will put the system into backup mode like pg_backup_start()
960 : * does, so that the backup is consistent even though we read directly from
961 : * the filesystem, bypassing the buffer cache.
962 : */
963 : void
964 CBC 143 : SendBaseBackup(BaseBackupCmd *cmd)
965 : {
966 : basebackup_options opt;
967 : bbsink *sink;
968 GIC 143 : SessionBackupState status = get_backup_status();
969 ECB :
970 GIC 143 : if (status == SESSION_BACKUP_RUNNING)
971 1 : ereport(ERROR,
972 : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
973 : errmsg("a backup is already in progress in this session")));
974 :
975 142 : parse_basebackup_options(cmd->options, &opt);
976 :
977 129 : WalSndSetState(WALSNDSTATE_BACKUP);
978 :
979 129 : if (update_process_title)
980 ECB : {
981 : char activitymsg[50];
982 :
983 GIC 129 : snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
984 ECB : opt.label);
985 GIC 129 : set_ps_display(activitymsg);
986 ECB : }
987 :
988 : /*
989 : * If the target is specifically 'client' then set up to stream the backup
990 : * to the client; otherwise, it's being sent someplace else and should not
991 : * be sent to the client. BaseBackupGetSink has the job of setting up a
992 : * sink to send the backup data wherever it needs to go.
993 : */
994 GIC 129 : sink = bbsink_copystream_new(opt.send_to_client);
995 CBC 129 : if (opt.target_handle != NULL)
996 GIC 13 : sink = BaseBackupGetSink(opt.target_handle, sink);
997 :
998 : /* Set up network throttling, if client requested it */
999 CBC 126 : if (opt.maxrate > 0)
1000 GIC 1 : sink = bbsink_throttle_new(sink, opt.maxrate);
1001 ECB :
1002 : /* Set up server-side compression, if client requested it */
1003 GIC 126 : if (opt.compression == PG_COMPRESSION_GZIP)
1004 2 : sink = bbsink_gzip_new(sink, &opt.compression_specification);
1005 124 : else if (opt.compression == PG_COMPRESSION_LZ4)
1006 2 : sink = bbsink_lz4_new(sink, &opt.compression_specification);
1007 122 : else if (opt.compression == PG_COMPRESSION_ZSTD)
1008 4 : sink = bbsink_zstd_new(sink, &opt.compression_specification);
1009 :
1010 ECB : /* Set up progress reporting. */
1011 CBC 126 : sink = bbsink_progress_new(sink, opt.progress);
1012 ECB :
1013 : /*
1014 : * Perform the base backup, but make sure we clean up the bbsink even if
1015 : * an error occurs.
1016 : */
1017 GIC 126 : PG_TRY();
1018 : {
1019 CBC 126 : perform_base_backup(&opt, sink);
1020 ECB : }
1021 CBC 4 : PG_FINALLY();
1022 ECB : {
1023 CBC 121 : bbsink_cleanup(sink);
1024 ECB : }
1025 GIC 121 : PG_END_TRY();
1026 117 : }
1027 ECB :
1028 : /*
1029 : * Inject a file with given name and content in the output tar stream.
1030 : */
1031 : static void
1032 GIC 170 : sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1033 ECB : backup_manifest_info *manifest)
1034 : {
1035 : struct stat statbuf;
1036 GIC 170 : int bytes_done = 0,
1037 ECB : len;
1038 : pg_checksum_context checksum_ctx;
1039 :
1040 GIC 170 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1041 LBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1042 ECB : filename);
1043 :
1044 GIC 170 : len = strlen(content);
1045 :
1046 : /*
1047 : * Construct a stat struct for the backup_label file we're injecting in
1048 ECB : * the tar.
1049 : */
1050 : /* Windows doesn't have the concept of uid and gid */
1051 : #ifdef WIN32
1052 : statbuf.st_uid = 0;
1053 : statbuf.st_gid = 0;
1054 : #else
1055 GIC 170 : statbuf.st_uid = geteuid();
1056 CBC 170 : statbuf.st_gid = getegid();
1057 EUB : #endif
1058 GIC 170 : statbuf.st_mtime = time(NULL);
1059 170 : statbuf.st_mode = pg_file_create_mode;
1060 CBC 170 : statbuf.st_size = len;
1061 :
1062 GIC 170 : _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1063 :
1064 170 : if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1065 UIC 0 : elog(ERROR, "could not update checksum of file \"%s\"",
1066 : filename);
1067 :
1068 GIC 298 : while (bytes_done < len)
1069 : {
1070 128 : size_t remaining = len - bytes_done;
1071 CBC 128 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1072 ECB :
1073 GIC 128 : memcpy(sink->bbs_buffer, content, nbytes);
1074 CBC 128 : bbsink_archive_contents(sink, nbytes);
1075 128 : bytes_done += nbytes;
1076 128 : content += nbytes;
1077 : }
1078 ECB :
1079 GIC 170 : _tarWritePadding(sink, len);
1080 ECB :
1081 GBC 170 : AddFileToBackupManifest(manifest, NULL, filename, len,
1082 GIC 170 : (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1083 170 : }
1084 ECB :
1085 : /*
1086 : * Include the tablespace directory pointed to by 'path' in the output tar
1087 : * stream. If 'sizeonly' is true, we just calculate a total length and return
1088 : * it, without actually sending anything.
1089 : *
1090 : * Only used to send auxiliary tablespaces, not PGDATA.
1091 : */
1092 : static int64
1093 GIC 32 : sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly,
1094 : backup_manifest_info *manifest)
1095 ECB : {
1096 : int64 size;
1097 : char pathbuf[MAXPGPATH];
1098 : struct stat statbuf;
1099 :
1100 : /*
1101 : * 'path' points to the tablespace location, but we only want to include
1102 : * the version directory in it that belongs to us.
1103 : */
1104 GIC 32 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1105 : TABLESPACE_VERSION_DIRECTORY);
1106 :
1107 : /*
1108 : * Store a directory entry in the tar file so we get the permissions
1109 ECB : * right.
1110 : */
1111 GIC 32 : if (lstat(pathbuf, &statbuf) != 0)
1112 : {
1113 UIC 0 : if (errno != ENOENT)
1114 0 : ereport(ERROR,
1115 : (errcode_for_file_access(),
1116 : errmsg("could not stat file or directory \"%s\": %m",
1117 : pathbuf)));
1118 :
1119 : /* If the tablespace went away while scanning, it's no error. */
1120 LBC 0 : return 0;
1121 : }
1122 :
1123 GIC 32 : size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1124 : sizeonly);
1125 :
1126 : /* Send all the files in the tablespace version directory */
1127 CBC 32 : size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1128 : spcoid);
1129 EUB :
1130 GBC 32 : return size;
1131 : }
1132 :
1133 : /*
1134 : * Include all files from the given directory in the output tar stream. If
1135 : * 'sizeonly' is true, we just calculate a total length and return it, without
1136 EUB : * actually sending anything.
1137 : *
1138 : * Omit any directory in the tablespaces list, to avoid backing up
1139 ECB : * tablespaces twice when they were created inside PGDATA.
1140 : *
1141 : * If sendtblspclinks is true, we need to include symlink
1142 : * information in the tar file. If not, we can skip that
1143 : * as it will be sent separately in the tablespace_map file.
1144 : */
1145 : static int64
1146 CBC 4316 : sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1147 : List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1148 : const char *spcoid)
1149 : {
1150 : DIR *dir;
1151 : struct dirent *de;
1152 : char pathbuf[MAXPGPATH * 2];
1153 : struct stat statbuf;
1154 GIC 4316 : int64 size = 0;
1155 : const char *lastDir; /* Split last dir from parent path. */
1156 4316 : bool isDbDir = false; /* Does this directory contain relations? */
1157 :
1158 : /*
1159 : * Determine if the current path is a database directory that can contain
1160 : * relations.
1161 : *
1162 ECB : * Start by finding the location of the delimiter between the parent path
1163 : * and the current path.
1164 : */
1165 GIC 4316 : lastDir = last_dir_separator(path);
1166 :
1167 : /* Does this path look like a database path (i.e. all digits)? */
1168 4316 : if (lastDir != NULL &&
1169 4064 : strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1170 ECB : {
1171 : /* Part of path that contains the parent directory. */
1172 CBC 811 : int parentPathLen = lastDir - path;
1173 :
1174 : /*
1175 : * Mark path as a database directory if the parent path is either
1176 : * $PGDATA/base or a tablespace version path.
1177 : */
1178 GIC 811 : if (strncmp(path, "./base", parentPathLen) == 0 ||
1179 54 : (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1180 32 : strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1181 ECB : TABLESPACE_VERSION_DIRECTORY,
1182 : sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1183 GIC 789 : isDbDir = true;
1184 ECB : }
1185 :
1186 GIC 4316 : dir = AllocateDir(path);
1187 263588 : while ((de = ReadDir(dir, path)) != NULL)
1188 ECB : {
1189 : int excludeIdx;
1190 : bool excludeFound;
1191 : ForkNumber relForkNum; /* Type of fork if file is a relation */
1192 : int relnumchars; /* Chars in filename that are the
1193 : * relnumber */
1194 :
1195 : /* Skip special stuff */
1196 CBC 259286 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1197 12211 : continue;
1198 :
1199 : /* Skip temporary files */
1200 250667 : if (strncmp(de->d_name,
1201 : PG_TEMP_FILE_PREFIX,
1202 : strlen(PG_TEMP_FILE_PREFIX)) == 0)
1203 247 : continue;
1204 ECB :
1205 : /*
1206 : * Check if the postmaster has signaled us to exit, and abort with an
1207 : * error in that case. The error handler further up will call
1208 : * do_pg_abort_backup() for us. Also check that if the backup was
1209 : * started while still in recovery, the server wasn't promoted.
1210 : * do_pg_backup_stop() will check that too, but it's better to stop
1211 : * the backup early than continue to the end and fail there.
1212 : */
1213 CBC 250420 : CHECK_FOR_INTERRUPTS();
1214 250415 : if (RecoveryInProgress() != backup_started_in_recovery)
1215 UIC 0 : ereport(ERROR,
1216 : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1217 ECB : errmsg("the standby was promoted during online backup"),
1218 : errhint("This means that the backup being taken is corrupt "
1219 : "and should not be used. "
1220 : "Try taking another online backup.")));
1221 :
1222 : /* Scan for files that should be excluded */
1223 GIC 250415 : excludeFound = false;
1224 2249800 : for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1225 : {
1226 2000384 : int cmplen = strlen(excludeFiles[excludeIdx].name);
1227 :
1228 2000384 : if (!excludeFiles[excludeIdx].match_prefix)
1229 1750103 : cmplen++;
1230 CBC 2000384 : if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1231 ECB : {
1232 GBC 999 : elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1233 GIC 999 : excludeFound = true;
1234 999 : break;
1235 : }
1236 : }
1237 :
1238 250415 : if (excludeFound)
1239 999 : continue;
1240 ECB :
1241 : /* Exclude all forks for unlogged tables except the init fork */
1242 GIC 475116 : if (isDbDir &&
1243 GNC 225700 : parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
1244 : &relForkNum))
1245 ECB : {
1246 : /* Never exclude init forks */
1247 CBC 224148 : if (relForkNum != INIT_FORKNUM)
1248 : {
1249 ECB : char initForkFile[MAXPGPATH];
1250 : char relNumber[OIDCHARS + 1];
1251 :
1252 : /*
1253 : * If any other type of fork, check if there is an init fork
1254 : * with the same RelFileNumber. If so, the file can be
1255 : * excluded.
1256 : */
1257 GNC 224079 : memcpy(relNumber, de->d_name, relnumchars);
1258 224079 : relNumber[relnumchars] = '\0';
1259 GIC 224079 : snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1260 : path, relNumber);
1261 ECB :
1262 GIC 224079 : if (lstat(initForkFile, &statbuf) == 0)
1263 : {
1264 69 : elog(DEBUG2,
1265 ECB : "unlogged relation file \"%s\" excluded from backup",
1266 : de->d_name);
1267 :
1268 GIC 69 : continue;
1269 : }
1270 : }
1271 : }
1272 :
1273 : /* Exclude temporary relations */
1274 249347 : if (isDbDir && looks_like_temp_rel_name(de->d_name))
1275 ECB : {
1276 CBC 40 : elog(DEBUG2,
1277 ECB : "temporary relation file \"%s\" excluded from backup",
1278 : de->d_name);
1279 :
1280 CBC 40 : continue;
1281 : }
1282 ECB :
1283 GIC 249307 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1284 :
1285 : /* Skip pg_control here to back up it last */
1286 CBC 249307 : if (strcmp(pathbuf, "./global/pg_control") == 0)
1287 GIC 247 : continue;
1288 :
1289 249060 : if (lstat(pathbuf, &statbuf) != 0)
1290 : {
1291 UIC 0 : if (errno != ENOENT)
1292 LBC 0 : ereport(ERROR,
1293 : (errcode_for_file_access(),
1294 ECB : errmsg("could not stat file or directory \"%s\": %m",
1295 : pathbuf)));
1296 :
1297 : /* If the file went away while scanning, it's not an error. */
1298 LBC 0 : continue;
1299 : }
1300 :
1301 ECB : /* Scan for directories whose contents should be excluded */
1302 GIC 249060 : excludeFound = false;
1303 1985521 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1304 ECB : {
1305 CBC 1738199 : if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1306 : {
1307 1738 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1308 GIC 1738 : convert_link_to_directory(pathbuf, &statbuf);
1309 GBC 1738 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1310 EUB : &statbuf, sizeonly);
1311 GIC 1738 : excludeFound = true;
1312 1738 : break;
1313 : }
1314 : }
1315 :
1316 GBC 249060 : if (excludeFound)
1317 GIC 1738 : continue;
1318 :
1319 : /*
1320 ECB : * We can skip pg_wal, the WAL segments need to be fetched from the
1321 : * WAL archive anyway. But include it as an empty directory anyway, so
1322 : * we get permissions right.
1323 : */
1324 GIC 247322 : if (strcmp(pathbuf, "./pg_wal") == 0)
1325 ECB : {
1326 : /* If pg_wal is a symlink, write it as a directory anyway */
1327 CBC 252 : convert_link_to_directory(pathbuf, &statbuf);
1328 GIC 252 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1329 ECB : &statbuf, sizeonly);
1330 :
1331 : /*
1332 : * Also send archive_status directory (by hackishly reusing
1333 : * statbuf from above ...).
1334 : */
1335 CBC 252 : size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1336 : &statbuf, sizeonly);
1337 :
1338 GIC 252 : continue; /* don't recurse into pg_wal */
1339 : }
1340 :
1341 : /* Allow symbolic links in pg_tblspc only */
1342 GNC 247070 : if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
1343 GIC 29 : {
1344 : char linkpath[MAXPGPATH];
1345 : int rllen;
1346 ECB :
1347 GIC 29 : rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1348 29 : if (rllen < 0)
1349 LBC 0 : ereport(ERROR,
1350 : (errcode_for_file_access(),
1351 : errmsg("could not read symbolic link \"%s\": %m",
1352 : pathbuf)));
1353 CBC 29 : if (rllen >= sizeof(linkpath))
1354 LBC 0 : ereport(ERROR,
1355 : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1356 : errmsg("symbolic link \"%s\" target is too long",
1357 : pathbuf)));
1358 CBC 29 : linkpath[rllen] = '\0';
1359 ECB :
1360 GBC 29 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1361 : &statbuf, sizeonly);
1362 ECB : }
1363 GIC 247041 : else if (S_ISDIR(statbuf.st_mode))
1364 ECB : {
1365 GIC 4056 : bool skip_this_dir = false;
1366 : ListCell *lc;
1367 :
1368 : /*
1369 : * Store a directory entry in the tar file so we can get the
1370 : * permissions right.
1371 ECB : */
1372 GIC 4056 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1373 : sizeonly);
1374 :
1375 : /*
1376 : * Call ourselves recursively for a directory, unless it happens
1377 : * to be a separate tablespace located within PGDATA.
1378 ECB : */
1379 GIC 8580 : foreach(lc, tablespaces)
1380 ECB : {
1381 GIC 4524 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1382 :
1383 : /*
1384 : * ti->rpath is the tablespace relative path within PGDATA, or
1385 : * NULL if the tablespace has been properly located somewhere
1386 : * else.
1387 : *
1388 : * Skip past the leading "./" in pathbuf when comparing.
1389 ECB : */
1390 GIC 4524 : if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1391 EUB : {
1392 UBC 0 : skip_this_dir = true;
1393 UIC 0 : break;
1394 : }
1395 : }
1396 :
1397 : /*
1398 : * skip sending directories inside pg_tblspc, if not required.
1399 ECB : */
1400 CBC 4056 : if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1401 GIC 24 : skip_this_dir = true;
1402 ECB :
1403 CBC 4056 : if (!skip_this_dir)
1404 GIC 4032 : size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1405 : sendtblspclinks, manifest, spcoid);
1406 ECB : }
1407 GIC 242985 : else if (S_ISREG(statbuf.st_mode))
1408 ECB : {
1409 GIC 242985 : bool sent = false;
1410 ECB :
1411 CBC 242985 : if (!sizeonly)
1412 229966 : sent = sendFile(sink, pathbuf, pathbuf + basepathlen + 1, &statbuf,
1413 GIC 110721 : true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1414 : manifest, spcoid);
1415 ECB :
1416 GIC 242984 : if (sent || sizeonly)
1417 : {
1418 ECB : /* Add size. */
1419 GIC 242984 : size += statbuf.st_size;
1420 :
1421 ECB : /* Pad to a multiple of the tar block size. */
1422 GIC 242984 : size += tarPaddingBytesRequired(statbuf.st_size);
1423 :
1424 ECB : /* Size of the header for the file. */
1425 GIC 242984 : size += TAR_BLOCK_SIZE;
1426 : }
1427 : }
1428 EUB : else
1429 UIC 0 : ereport(WARNING,
1430 : (errmsg("skipping special file \"%s\"", pathbuf)));
1431 ECB : }
1432 CBC 4302 : FreeDir(dir);
1433 GIC 4302 : return size;
1434 : }
1435 :
1436 : /*
1437 : * Check if a file should have its checksum validated.
1438 : * We validate checksums on files in regular tablespaces
1439 : * (including global and default) only, and in those there
1440 : * are some files that are explicitly excluded.
1441 : */
1442 ECB : static bool
1443 GIC 28387 : is_checksummed_file(const char *fullpath, const char *filename)
1444 : {
1445 ECB : /* Check that the file is in a tablespace */
1446 CBC 28387 : if (strncmp(fullpath, "./global/", 9) == 0 ||
1447 26618 : strncmp(fullpath, "./base/", 7) == 0 ||
1448 GIC 329 : strncmp(fullpath, "/", 1) == 0)
1449 : {
1450 : int excludeIdx;
1451 :
1452 ECB : /* Compare file against noChecksumFiles skip list */
1453 GIC 139874 : for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1454 ECB : {
1455 GIC 112016 : int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1456 ECB :
1457 CBC 112016 : if (!noChecksumFiles[excludeIdx].match_prefix)
1458 84071 : cmplen++;
1459 GIC 112016 : if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1460 ECB : cmplen) == 0)
1461 GIC 205 : return false;
1462 : }
1463 ECB :
1464 GIC 27858 : return true;
1465 : }
1466 ECB : else
1467 GIC 324 : return false;
1468 : }
1469 :
1470 : /*
1471 : * Given the member, write the TAR header & send the file.
1472 : *
1473 : * If 'missing_ok' is true, will not throw an error if the file is not found.
1474 ECB : *
1475 : * If dboid is anything other than InvalidOid then any checksum failures
1476 : * detected will get reported to the cumulative stats system.
1477 : *
1478 : * Returns true if the file was successfully sent, false if 'missing_ok',
1479 : * and the file did not exist.
1480 : */
1481 : static bool
1482 CBC 119365 : sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1483 : struct stat *statbuf, bool missing_ok, Oid dboid,
1484 : backup_manifest_info *manifest, const char *spcoid)
1485 ECB : {
1486 : int fd;
1487 GIC 119365 : BlockNumber blkno = 0;
1488 CBC 119365 : bool block_retry = false;
1489 : uint16 checksum;
1490 119365 : int checksum_failures = 0;
1491 : off_t cnt;
1492 : int i;
1493 119365 : pgoff_t len = 0;
1494 EUB : char *page;
1495 : PageHeader phdr;
1496 GIC 119365 : int segmentno = 0;
1497 ECB : char *segmentpath;
1498 CBC 119365 : bool verify_checksum = false;
1499 : pg_checksum_context checksum_ctx;
1500 EUB :
1501 GBC 119365 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1502 UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1503 : readfilename);
1504 :
1505 GIC 119365 : fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1506 119365 : if (fd < 0)
1507 ECB : {
1508 UIC 0 : if (errno == ENOENT && missing_ok)
1509 LBC 0 : return false;
1510 UIC 0 : ereport(ERROR,
1511 : (errcode_for_file_access(),
1512 : errmsg("could not open file \"%s\": %m", readfilename)));
1513 : }
1514 :
1515 GIC 119365 : _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1516 :
1517 119364 : if (!noverify_checksums && DataChecksumsEnabled())
1518 ECB : {
1519 : char *filename;
1520 :
1521 : /*
1522 : * Get the filename (excluding path). As last_dir_separator()
1523 : * includes the last directory separator, we chop that off by
1524 : * incrementing the pointer.
1525 : */
1526 GIC 28387 : filename = last_dir_separator(readfilename) + 1;
1527 :
1528 CBC 28387 : if (is_checksummed_file(readfilename, filename))
1529 ECB : {
1530 GIC 27858 : verify_checksum = true;
1531 EUB :
1532 : /*
1533 : * Cut off at the segment boundary (".") to get the segment number
1534 : * in order to mix it into the checksum.
1535 : */
1536 GIC 27858 : segmentpath = strstr(filename, ".");
1537 27858 : if (segmentpath != NULL)
1538 : {
1539 UIC 0 : segmentno = atoi(segmentpath + 1);
1540 0 : if (segmentno == 0)
1541 0 : ereport(ERROR,
1542 : (errmsg("invalid segment number %d in file \"%s\"",
1543 : segmentno, filename)));
1544 : }
1545 : }
1546 ECB : }
1547 :
1548 : /*
1549 : * Loop until we read the amount of data the caller told us to expect. The
1550 : * file could be longer, if it was extended while we were sending it, but
1551 : * for a base backup we can ignore such extended data. It will be restored
1552 : * from WAL.
1553 : */
1554 GIC 265468 : while (len < statbuf->st_size)
1555 : {
1556 146104 : size_t remaining = statbuf->st_size - len;
1557 :
1558 : /* Try to read some more data. */
1559 292208 : cnt = basebackup_read_file(fd, sink->bbs_buffer,
1560 146104 : Min(sink->bbs_buffer_length, remaining),
1561 ECB : len, readfilename, true);
1562 :
1563 : /*
1564 : * The checksums are verified at block level, so we iterate over the
1565 EUB : * buffer in chunks of BLCKSZ, after making sure that
1566 : * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1567 : * BLCKSZ bytes.
1568 : */
1569 GIC 146104 : Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1570 EUB :
1571 GIC 146104 : if (verify_checksum && (cnt % BLCKSZ != 0))
1572 : {
1573 LBC 0 : ereport(WARNING,
1574 : (errmsg("could not verify checksum in file \"%s\", block "
1575 ECB : "%u: read buffer size %d and page size %d "
1576 : "differ",
1577 : readfilename, blkno, (int) cnt, BLCKSZ)));
1578 UIC 0 : verify_checksum = false;
1579 : }
1580 :
1581 GIC 146104 : if (verify_checksum)
1582 : {
1583 118694 : for (i = 0; i < cnt / BLCKSZ; i++)
1584 : {
1585 84496 : page = sink->bbs_buffer + BLCKSZ * i;
1586 :
1587 ECB : /*
1588 : * Only check pages which have not been modified since the
1589 : * start of the base backup. Otherwise, they might have been
1590 : * written only halfway and the checksum would not be valid.
1591 : * However, replaying WAL would reinstate the correct page in
1592 : * this case. We also skip completely new pages, since they
1593 : * don't have a checksum yet.
1594 : */
1595 GIC 84496 : if (!PageIsNew(page) && PageGetLSN(page) < sink->bbs_state->startptr)
1596 : {
1597 84496 : checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1598 84496 : phdr = (PageHeader) page;
1599 84496 : if (phdr->pd_checksum != checksum)
1600 : {
1601 : /*
1602 : * Retry the block on the first failure. It's
1603 : * possible that we read the first 4K page of the
1604 : * block just before postgres updated the entire block
1605 : * so it ends up looking torn to us. If, before we
1606 : * retry the read, the concurrent write of the block
1607 : * finishes, the page LSN will be updated and we'll
1608 : * realize that we should ignore this block.
1609 : *
1610 : * There's no guarantee that this will actually
1611 : * happen, though: the torn write could take an
1612 : * arbitrarily long time to complete. Retrying multiple
1613 : * times wouldn't fix this problem, either, though
1614 : * it would reduce the chances of it happening in
1615 : * practice. The only real fix here seems to be to
1616 : * have some kind of interlock that allows us to wait
1617 : * until we can be certain that no write to the block
1618 : * is in progress. Since we don't have any such thing
1619 : * right now, we just do this and hope for the best.
1620 : */
1621 28 : if (block_retry == false)
1622 14 : {
1623 ECB : int reread_cnt;
1624 :
1625 : /* Reread the failed block */
1626 : reread_cnt =
1627 GIC 14 : basebackup_read_file(fd,
1628 14 : sink->bbs_buffer + BLCKSZ * i,
1629 CBC 14 : BLCKSZ, len + BLCKSZ * i,
1630 ECB : readfilename,
1631 : false);
1632 GIC 14 : if (reread_cnt == 0)
1633 : {
1634 ECB : /*
1635 : * If we hit end-of-file, a concurrent
1636 : * truncation must have occurred, so break out
1637 : * of this loop just as if the initial fread()
1638 : * returned 0. We'll drop through to the same
1639 : * code that handles that case. (We must fix
1640 : * up cnt first, though.)
1641 : */
1642 UIC 0 : cnt = BLCKSZ * i;
1643 0 : break;
1644 EUB : }
1645 :
1646 : /* Set flag so we know a retry was attempted */
1647 GIC 14 : block_retry = true;
1648 :
1649 ECB : /* Reset loop to validate the block again */
1650 GIC 14 : i--;
1651 14 : continue;
1652 ECB : }
1653 :
1654 GIC 14 : checksum_failures++;
1655 :
1656 CBC 14 : if (checksum_failures <= 5)
1657 GIC 12 : ereport(WARNING,
1658 ECB : (errmsg("checksum verification failed in "
1659 : "file \"%s\", block %u: calculated "
1660 : "%X but expected %X",
1661 : readfilename, blkno, checksum,
1662 : phdr->pd_checksum)));
1663 GIC 14 : if (checksum_failures == 5)
1664 2 : ereport(WARNING,
1665 ECB : (errmsg("further checksum verification "
1666 : "failures in file \"%s\" will not "
1667 : "be reported", readfilename)));
1668 : }
1669 : }
1670 GIC 84482 : block_retry = false;
1671 84482 : blkno++;
1672 ECB : }
1673 : }
1674 :
1675 : /*
1676 : * If we hit end-of-file, a concurrent truncation must have occurred.
1677 : * That's not an error condition, because WAL replay will fix things
1678 : * up.
1679 : */
1680 GIC 146104 : if (cnt == 0)
1681 UIC 0 : break;
1682 ECB :
1683 EUB : /* Archive the data we just read. */
1684 GIC 146104 : bbsink_archive_contents(sink, cnt);
1685 :
1686 ECB : /* Also feed it to the checksum machinery. */
1687 GIC 146104 : if (pg_checksum_update(&checksum_ctx,
1688 146104 : (uint8 *) sink->bbs_buffer, cnt) < 0)
1689 LBC 0 : elog(ERROR, "could not update checksum of base backup");
1690 ECB :
1691 GBC 146104 : len += cnt;
1692 : }
1693 ECB :
1694 : /* If the file was truncated while we were sending it, pad it with zeros */
1695 GIC 119364 : while (len < statbuf->st_size)
1696 : {
1697 LBC 0 : size_t remaining = statbuf->st_size - len;
1698 UIC 0 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1699 EUB :
1700 UBC 0 : MemSet(sink->bbs_buffer, 0, nbytes);
1701 UIC 0 : if (pg_checksum_update(&checksum_ctx,
1702 UBC 0 : (uint8 *) sink->bbs_buffer,
1703 EUB : nbytes) < 0)
1704 UBC 0 : elog(ERROR, "could not update checksum of base backup");
1705 UIC 0 : bbsink_archive_contents(sink, nbytes);
1706 UBC 0 : len += nbytes;
1707 EUB : }
1708 :
1709 : /*
1710 : * Pad to a block boundary, per tar format requirements. (This small piece
1711 : * of data is probably not worth throttling, and is not checksummed
1712 : * because it's not actually part of the file.)
1713 : */
1714 GIC 119364 : _tarWritePadding(sink, len);
1715 :
1716 CBC 119364 : CloseTransientFile(fd);
1717 :
1718 119364 : if (checksum_failures > 1)
1719 : {
1720 2 : ereport(WARNING,
1721 : (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1722 ECB : "file \"%s\" has a total of %d checksum verification failures",
1723 : checksum_failures,
1724 : readfilename, checksum_failures)));
1725 :
1726 GIC 2 : pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1727 : }
1728 ECB :
1729 GIC 119364 : total_checksum_failures += checksum_failures;
1730 :
1731 CBC 119364 : AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1732 GIC 119364 : (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1733 ECB :
1734 CBC 119364 : return true;
1735 : }
1736 ECB :
1737 : static int64
1738 GIC 125914 : _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
1739 : struct stat *statbuf, bool sizeonly)
1740 ECB : {
1741 : enum tarError rc;
1742 :
1743 GIC 125914 : if (!sizeonly)
1744 : {
1745 ECB : /*
1746 : * As of this writing, the smallest supported block size is 1kB, which
1747 : * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
1748 : * multiple of BLCKSZ, it should be safe to assume that the buffer is
1749 : * large enough to fit an entire tar block. We double-check by means
1750 : * of these assertions.
1751 : */
1752 : StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
1753 : "BLCKSZ too small for tar block");
1754 GIC 122684 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
1755 :
1756 CBC 122684 : rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
1757 : statbuf->st_size, statbuf->st_mode,
1758 ECB : statbuf->st_uid, statbuf->st_gid,
1759 : statbuf->st_mtime);
1760 :
1761 GIC 122684 : switch (rc)
1762 : {
1763 CBC 122683 : case TAR_OK:
1764 GIC 122683 : break;
1765 CBC 1 : case TAR_NAME_TOO_LONG:
1766 1 : ereport(ERROR,
1767 ECB : (errmsg("file name too long for tar format: \"%s\"",
1768 : filename)));
1769 : break;
1770 UIC 0 : case TAR_SYMLINK_TOO_LONG:
1771 0 : ereport(ERROR,
1772 EUB : (errmsg("symbolic link target too long for tar format: "
1773 : "file name \"%s\", target \"%s\"",
1774 : filename, linktarget)));
1775 : break;
1776 UIC 0 : default:
1777 0 : elog(ERROR, "unrecognized tar error: %d", rc);
1778 EUB : }
1779 :
1780 GIC 122683 : bbsink_archive_contents(sink, TAR_BLOCK_SIZE);
1781 : }
1782 ECB :
1783 GIC 125913 : return TAR_BLOCK_SIZE;
1784 : }
1785 ECB :
1786 : /*
1787 : * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
1788 : */
1789 : static void
1790 GIC 119534 : _tarWritePadding(bbsink *sink, int len)
1791 : {
1792 CBC 119534 : int pad = tarPaddingBytesRequired(len);
1793 :
1794 ECB : /*
1795 : * As in _tarWriteHeader, it should be safe to assume that the buffer is
1796 : * large enough that we don't need to do this in multiple chunks.
1797 : */
1798 GIC 119534 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
1799 119534 : Assert(pad <= TAR_BLOCK_SIZE);
1800 ECB :
1801 CBC 119534 : if (pad > 0)
1802 : {
1803 22769 : MemSet(sink->bbs_buffer, 0, pad);
1804 GIC 1768 : bbsink_archive_contents(sink, pad);
1805 ECB : }
1806 CBC 119534 : }
1807 :
1808 ECB : /*
1809 : * If the entry in statbuf is a link, then adjust statbuf to make it look like a
1810 : * directory, so that it will be written that way.
1811 : */
1812 : static void
1813 GIC 1990 : convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
1814 : {
1815 ECB : /* If symlink, write it as a directory anyway */
1816 GIC 1990 : if (S_ISLNK(statbuf->st_mode))
1817 66 : statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1818 1990 : }
1819 :
1820 : /*
1821 : * Read some data from a file, setting a wait event and reporting any error
1822 : * encountered.
1823 : *
1824 : * If partial_read_ok is false, also report an error if the number of bytes
1825 : * read is not equal to the number of bytes requested.
1826 : *
1827 : * Returns the number of bytes read.
1828 ECB : */
1829 : static int
1830 GIC 156358 : basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
1831 : const char *filename, bool partial_read_ok)
1832 : {
1833 ECB : int rc;
1834 :
1835 CBC 156358 : pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
1836 GIC 156358 : rc = pg_pread(fd, buf, nbytes, offset);
1837 CBC 156358 : pgstat_report_wait_end();
1838 EUB :
1839 GIC 156358 : if (rc < 0)
1840 UIC 0 : ereport(ERROR,
1841 ECB : (errcode_for_file_access(),
1842 EUB : errmsg("could not read file \"%s\": %m", filename)));
1843 GIC 156358 : if (!partial_read_ok && rc > 0 && rc != nbytes)
1844 UIC 0 : ereport(ERROR,
1845 : (errcode_for_file_access(),
1846 : errmsg("could not read file \"%s\": read %d of %zu",
1847 ECB : filename, rc, nbytes)));
1848 :
1849 GIC 156358 : return rc;
1850 : }
|
