Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * basebackup.c
4 : * code for taking a base backup and streaming it to a standby
5 : *
6 : * Portions Copyright (c) 2010-2023, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/backup/basebackup.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include <sys/stat.h>
16 : #include <unistd.h>
17 : #include <time.h>
18 :
19 : #include "access/xlog_internal.h"
20 : #include "access/xlogbackup.h"
21 : #include "backup/backup_manifest.h"
22 : #include "backup/basebackup.h"
23 : #include "backup/basebackup_sink.h"
24 : #include "backup/basebackup_target.h"
25 : #include "commands/defrem.h"
26 : #include "common/compression.h"
27 : #include "common/file_perm.h"
28 : #include "lib/stringinfo.h"
29 : #include "miscadmin.h"
30 : #include "nodes/pg_list.h"
31 : #include "pgstat.h"
32 : #include "pgtar.h"
33 : #include "port.h"
34 : #include "postmaster/syslogger.h"
35 : #include "replication/walsender.h"
36 : #include "replication/walsender_private.h"
37 : #include "storage/bufpage.h"
38 : #include "storage/checksum.h"
39 : #include "storage/dsm_impl.h"
40 : #include "storage/fd.h"
41 : #include "storage/ipc.h"
42 : #include "storage/reinit.h"
43 : #include "utils/builtins.h"
44 : #include "utils/guc.h"
45 : #include "utils/ps_status.h"
46 : #include "utils/relcache.h"
47 : #include "utils/resowner.h"
48 : #include "utils/timestamp.h"
49 :
50 : /*
51 : * How much data do we want to send in one CopyData message? Note that
52 : * this may also result in reading the underlying files in chunks of this
53 : * size.
54 : *
55 : * NB: The buffer size is required to be a multiple of the system block
56 : * size, so use that value instead if it's bigger than our preference.
57 : */
58 : #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
59 :
60 : typedef struct
61 : {
62 : const char *label;
63 : bool progress;
64 : bool fastcheckpoint;
65 : bool nowait;
66 : bool includewal;
67 : uint32 maxrate;
68 : bool sendtblspcmapfile;
69 : bool send_to_client;
70 : bool use_copytblspc;
71 : BaseBackupTargetHandle *target_handle;
72 : backup_manifest_option manifest;
73 : pg_compress_algorithm compression;
74 : pg_compress_specification compression_specification;
75 : pg_checksum_type manifest_checksum_type;
76 : } basebackup_options;
77 :
78 : static int64 sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly,
79 : struct backup_manifest_info *manifest);
80 : static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
81 : List *tablespaces, bool sendtblspclinks,
82 : backup_manifest_info *manifest, const char *spcoid);
83 : static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
84 : struct stat *statbuf, bool missing_ok, Oid dboid,
85 : backup_manifest_info *manifest, const char *spcoid);
86 : static void sendFileWithContent(bbsink *sink, const char *filename,
87 : const char *content,
88 : backup_manifest_info *manifest);
89 : static int64 _tarWriteHeader(bbsink *sink, const char *filename,
90 : const char *linktarget, struct stat *statbuf,
91 : bool sizeonly);
92 : static void _tarWritePadding(bbsink *sink, int len);
93 : static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
94 : static void perform_base_backup(basebackup_options *opt, bbsink *sink);
95 : static void parse_basebackup_options(List *options, basebackup_options *opt);
96 : static int compareWalFileNames(const ListCell *a, const ListCell *b);
97 : static bool is_checksummed_file(const char *fullpath, const char *filename);
98 : static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
99 : const char *filename, bool partial_read_ok);
100 :
101 : /* Was the backup currently in-progress initiated in recovery mode? */
102 : static bool backup_started_in_recovery = false;
103 :
104 : /* Total number of checksum failures during base backup. */
105 : static long long int total_checksum_failures;
106 :
107 : /* Do not verify checksums. */
108 : static bool noverify_checksums = false;
109 :
110 : /*
111 : * Definition of one element part of an exclusion list, used for paths part
112 : * of checksum validation or base backups. "name" is the name of the file
113 : * or path to check for exclusion. If "match_prefix" is true, any items
114 : * matching the name as prefix are excluded.
115 : */
116 : struct exclude_list_item
117 : {
118 : const char *name;
119 : bool match_prefix;
120 : };
121 :
122 : /*
123 : * The contents of these directories are removed or recreated during server
124 : * start so they are not included in backups. The directories themselves are
125 : * kept and included as empty to preserve access permissions.
126 : *
127 : * Note: this list should be kept in sync with the filter lists in pg_rewind's
128 : * filemap.c.
129 : */
130 : static const char *const excludeDirContents[] =
131 : {
132 : /*
133 : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
134 : * because extensions like pg_stat_statements store data there.
135 : */
136 : PG_STAT_TMP_DIR,
137 :
138 : /*
139 : * It is generally not useful to backup the contents of this directory
140 : * even if the intention is to restore to another primary. See backup.sgml
141 : * for a more detailed description.
142 : */
143 : "pg_replslot",
144 :
145 : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
146 : PG_DYNSHMEM_DIR,
147 :
148 : /* Contents removed on startup, see AsyncShmemInit(). */
149 : "pg_notify",
150 :
151 : /*
152 : * Old contents are loaded for possible debugging but are not required for
153 : * normal operation, see SerialInit().
154 : */
155 : "pg_serial",
156 :
157 : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
158 : "pg_snapshots",
159 :
160 : /* Contents zeroed on startup, see StartupSUBTRANS(). */
161 : "pg_subtrans",
162 :
163 : /* end of list */
164 : NULL
165 : };
166 :
167 : /*
168 : * List of files excluded from backups.
169 : */
170 : static const struct exclude_list_item excludeFiles[] =
171 : {
172 : /* Skip auto conf temporary file. */
173 : {PG_AUTOCONF_FILENAME ".tmp", false},
174 :
175 : /* Skip current log file temporary file */
176 : {LOG_METAINFO_DATAFILE_TMP, false},
177 :
178 : /*
179 : * Skip relation cache because it is rebuilt on startup. This includes
180 : * temporary files.
181 : */
182 : {RELCACHE_INIT_FILENAME, true},
183 :
184 : /*
185 : * backup_label and tablespace_map should not exist in a running cluster
186 : * capable of doing an online backup, but exclude them just in case.
187 : */
188 : {BACKUP_LABEL_FILE, false},
189 : {TABLESPACE_MAP, false},
190 :
191 : /*
192 : * If there's a backup_manifest, it belongs to a backup that was used to
193 : * start this server. It is *not* correct for this backup. Our
194 : * backup_manifest is injected into the backup separately if users want
195 : * it.
196 : */
197 : {"backup_manifest", false},
198 :
199 : {"postmaster.pid", false},
200 : {"postmaster.opts", false},
201 :
202 : /* end of list */
203 : {NULL, false}
204 : };
205 :
206 : /*
207 : * List of files excluded from checksum validation.
208 : *
209 : * Note: this list should be kept in sync with what pg_checksums.c
210 : * includes.
211 : */
212 : static const struct exclude_list_item noChecksumFiles[] = {
213 : {"pg_control", false},
214 : {"pg_filenode.map", false},
215 : {"pg_internal.init", true},
216 : {"PG_VERSION", false},
217 : #ifdef EXEC_BACKEND
218 : {"config_exec_params", true},
219 : #endif
220 : {NULL, false}
221 : };
222 :
223 : /*
224 : * Actually do a base backup for the specified tablespaces.
225 : *
226 : * This is split out mainly to avoid complaints about "variable might be
227 : * clobbered by longjmp" from stupider versions of gcc.
228 : */
229 : static void
520 rhaas 230 GIC 126 : perform_base_backup(basebackup_options *opt, bbsink *sink)
231 : {
520 rhaas 232 ECB : bbsink_state state;
233 : XLogRecPtr endptr;
234 : TimeLineID endtli;
235 : backup_manifest_info manifest;
236 : BackupState *backup_state;
237 : StringInfo tablespace_map;
238 :
239 : /* Initial backup state, insofar as we know it now. */
520 rhaas 240 GIC 126 : state.tablespaces = NIL;
241 126 : state.tablespace_num = 0;
520 rhaas 242 CBC 126 : state.bytes_done = 0;
243 126 : state.bytes_total = 0;
244 126 : state.bytes_total_is_valid = false;
1111 fujii 245 ECB :
1101 rhaas 246 : /* we're going to use a BufFile, so we need a ResourceOwner */
1101 rhaas 247 GIC 126 : Assert(CurrentResourceOwner == NULL);
248 126 : CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
1101 rhaas 249 ECB :
3769 heikki.linnakangas 250 CBC 126 : backup_started_in_recovery = RecoveryInProgress();
251 :
1081 rhaas 252 126 : InitializeBackupManifest(&manifest, opt->manifest,
253 : opt->manifest_checksum_type);
254 :
1832 magnus 255 126 : total_checksum_failures = 0;
256 :
257 : /* Allocate backup related varilables. */
195 michael 258 GNC 126 : backup_state = (BackupState *) palloc0(sizeof(BackupState));
259 126 : tablespace_map = makeStringInfo();
260 :
520 rhaas 261 GIC 126 : basebackup_progress_wait_checkpoint();
195 michael 262 GNC 126 : do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
263 : backup_state, tablespace_map);
264 :
265 126 : state.startptr = backup_state->startpoint;
266 126 : state.starttli = backup_state->starttli;
2878 bruce 267 ECB :
268 : /*
269 : * Once do_pg_backup_start has been called, ensure that any failure causes
270 : * us to abort the backup so we don't "leak" a backup counter. For this
368 sfrost 271 : * reason, *all* functionality between do_pg_backup_start() and the end of
272 : * do_pg_backup_stop() should be inside the error cleanup block!
273 : */
274 :
1207 rhaas 275 GIC 126 : PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
276 : {
277 : ListCell *lc;
278 : tablespaceinfo *newti;
279 :
4452 magnus 280 ECB : /* Add a node for the base directory at the end */
186 drowley 281 GNC 126 : newti = palloc0(sizeof(tablespaceinfo));
282 126 : newti->size = -1;
283 126 : state.tablespaces = lappend(state.tablespaces, newti);
284 :
285 : /*
1132 fujii 286 ECB : * Calculate the total backup size by summing up the size of each
287 : * tablespace
288 : */
1132 fujii 289 GIC 126 : if (opt->progress)
290 : {
520 rhaas 291 126 : basebackup_progress_estimate_backup_size();
292 :
293 268 : foreach(lc, state.tablespaces)
1132 fujii 294 ECB : {
1132 fujii 295 GIC 142 : tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
1132 fujii 296 ECB :
1026 rhaas 297 GIC 142 : if (tmp->path == NULL)
520 rhaas 298 CBC 126 : tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
299 : true, NULL, NULL);
1026 rhaas 300 ECB : else
520 rhaas 301 GIC 16 : tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
1026 rhaas 302 ECB : NULL);
520 rhaas 303 CBC 142 : state.bytes_total += tmp->size;
304 : }
520 rhaas 305 GIC 126 : state.bytes_total_is_valid = true;
1132 fujii 306 ECB : }
307 :
520 rhaas 308 : /* notify basebackup sink about start of backup */
520 rhaas 309 GIC 126 : bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
3328 alvherre 310 ECB :
311 : /* Send off our tablespaces one by one */
520 rhaas 312 GIC 262 : foreach(lc, state.tablespaces)
313 : {
4471 tgl 314 CBC 142 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
315 :
4092 simon 316 GIC 142 : if (ti->path == NULL)
4092 simon 317 ECB : {
318 : struct stat statbuf;
697 tgl 319 CBC 126 : bool sendtblspclinks = true;
320 : char *backup_label;
321 :
520 rhaas 322 126 : bbsink_begin_archive(sink, "base.tar");
323 :
324 : /* In the main tar, include the backup_label first... */
195 michael 325 GNC 126 : backup_label = build_backup_content(backup_state, false);
326 126 : sendFileWithContent(sink, BACKUP_LABEL_FILE,
327 : backup_label, &manifest);
328 126 : pfree(backup_label);
329 :
1026 rhaas 330 ECB : /* Then the tablespace_map file, if required... */
1026 rhaas 331 GIC 126 : if (opt->sendtblspcmapfile)
332 : {
195 michael 333 GNC 24 : sendFileWithContent(sink, TABLESPACE_MAP,
334 24 : tablespace_map->data, &manifest);
1026 rhaas 335 GIC 24 : sendtblspclinks = false;
2889 andrew 336 ECB : }
337 :
338 : /* Then the bulk of the files... */
520 rhaas 339 CBC 126 : sendDir(sink, ".", 1, false, state.tablespaces,
340 : sendtblspclinks, &manifest, NULL);
3667 heikki.linnakangas 341 ECB :
342 : /* ... and pg_control after everything else. */
4092 simon 343 CBC 120 : if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
4092 simon 344 UIC 0 : ereport(ERROR,
345 : (errcode_for_file_access(),
346 : errmsg("could not stat file \"%s\": %m",
4092 simon 347 ECB : XLOG_CONTROL_FILE)));
520 rhaas 348 GIC 120 : sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
349 : false, InvalidOid, &manifest, NULL);
350 : }
3667 heikki.linnakangas 351 ECB : else
520 rhaas 352 EUB : {
520 rhaas 353 GIC 16 : char *archive_name = psprintf("%s.tar", ti->oid);
354 :
355 16 : bbsink_begin_archive(sink, archive_name);
520 rhaas 356 ECB :
520 rhaas 357 GIC 16 : sendTablespace(sink, ti->path, ti->oid, false, &manifest);
358 : }
359 :
360 : /*
4452 magnus 361 ECB : * If we're including WAL, and this is the main data directory we
362 : * don't treat this as the end of the tablespace. Instead, we will
520 rhaas 363 : * include the xlog files below and stop afterwards. This is safe
364 : * since the main data directory is always sent *last*.
4452 magnus 365 : */
4452 magnus 366 GIC 136 : if (opt->includewal && ti->path == NULL)
367 : {
520 rhaas 368 20 : Assert(lnext(state.tablespaces, lc) == NULL);
369 : }
370 : else
371 : {
372 : /* Properly terminate the tarfile. */
373 : StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ,
516 rhaas 374 ECB : "BLCKSZ too small for 2 tar blocks");
516 rhaas 375 GIC 116 : memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
516 rhaas 376 CBC 116 : bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
377 :
378 : /* OK, that's the end of the archive. */
520 rhaas 379 GIC 116 : bbsink_end_archive(sink);
380 : }
381 : }
382 :
520 rhaas 383 CBC 120 : basebackup_progress_wait_wal_archive(&state);
195 michael 384 GNC 120 : do_pg_backup_stop(backup_state, !opt->nowait);
385 :
386 120 : endptr = backup_state->stoppoint;
387 120 : endtli = backup_state->stoptli;
388 :
389 : /* Deallocate backup-related variables. */
390 120 : pfree(tablespace_map->data);
391 120 : pfree(tablespace_map);
392 120 : pfree(backup_state);
393 : }
1207 rhaas 394 GIC 121 : PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
4471 tgl 395 ECB :
396 :
4452 magnus 397 GIC 120 : if (opt->includewal)
398 : {
4452 magnus 399 ECB : /*
400 : * We've left the last tar file "open", so we can now append the
401 : * required WAL files to it.
402 : */
3748 heikki.linnakangas 403 : char pathbuf[MAXPGPATH];
404 : XLogSegNo segno;
405 : XLogSegNo startsegno;
406 : XLogSegNo endsegno;
4452 magnus 407 : struct stat statbuf;
3748 heikki.linnakangas 408 CBC 20 : List *historyFileList = NIL;
3748 heikki.linnakangas 409 GIC 20 : List *walFileList = NIL;
3748 heikki.linnakangas 410 ECB : char firstoff[MAXFNAMELEN];
411 : char lastoff[MAXFNAMELEN];
412 : DIR *dir;
413 : struct dirent *de;
414 : ListCell *lc;
415 : TimeLineID tli;
416 :
520 rhaas 417 GIC 20 : basebackup_progress_transfer_wal();
418 :
419 : /*
420 : * I'd rather not worry about timelines here, so scan pg_wal and
421 : * include all WAL files in the range between 'startptr' and 'endptr',
422 : * regardless of the timeline the file is stamped with. If there are
423 : * some spurious WAL files belonging to timelines that don't belong in
3602 bruce 424 ECB : * this server's history, they will be included too. Normally there
3748 heikki.linnakangas 425 : * shouldn't be such files, but if there are, there's little harm in
426 : * including them.
427 : */
520 rhaas 428 GIC 20 : XLByteToSeg(state.startptr, startsegno, wal_segment_size);
429 20 : XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
2028 andres 430 20 : XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
527 rhaas 431 20 : XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
432 :
2362 rhaas 433 CBC 20 : dir = AllocateDir("pg_wal");
2362 rhaas 434 GIC 123 : while ((de = ReadDir(dir, "pg_wal")) != NULL)
435 : {
436 : /* Does it look like a WAL segment, and is it in the range? */
2893 heikki.linnakangas 437 103 : if (IsXLogFileName(de->d_name) &&
3748 438 43 : strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
439 43 : strcmp(de->d_name + 8, lastoff + 8) <= 0)
440 : {
441 20 : walFileList = lappend(walFileList, pstrdup(de->d_name));
442 : }
443 : /* Does it look like a timeline history file? */
2893 heikki.linnakangas 444 CBC 83 : else if (IsTLHistoryFileName(de->d_name))
3748 heikki.linnakangas 445 ECB : {
3748 heikki.linnakangas 446 LBC 0 : historyFileList = lappend(historyFileList, pstrdup(de->d_name));
3748 heikki.linnakangas 447 ECB : }
448 : }
3748 heikki.linnakangas 449 CBC 20 : FreeDir(dir);
4452 magnus 450 ECB :
451 : /*
452 : * Before we go any further, check that none of the WAL segments we
3748 heikki.linnakangas 453 : * need were removed.
454 : */
520 rhaas 455 CBC 20 : CheckXLogRemoved(startsegno, state.starttli);
456 :
3748 heikki.linnakangas 457 ECB : /*
458 : * Sort the WAL filenames. We want to send the files in order from
459 : * oldest to newest, to reduce the chance that a file is recycled
1363 tgl 460 : * before we get a chance to send it over.
461 : */
1363 tgl 462 GBC 20 : list_sort(walFileList, compareWalFileNames);
463 :
464 : /*
2153 bruce 465 ECB : * There must be at least one xlog file in the pg_wal directory, since
466 : * we are doing backup-including-xlog.
467 : */
1363 tgl 468 GIC 20 : if (walFileList == NIL)
3515 magnus 469 UIC 0 : ereport(ERROR,
470 : (errmsg("could not find any WAL files")));
3515 magnus 471 ECB :
472 : /*
473 : * Sanity check: the first and last segment should cover startptr and
474 : * endptr, with no gaps in between.
475 : */
1363 tgl 476 GIC 20 : XLogFromFileName((char *) linitial(walFileList),
477 : &tli, &segno, wal_segment_size);
3748 heikki.linnakangas 478 CBC 20 : if (segno != startsegno)
479 : {
480 : char startfname[MAXFNAMELEN];
481 :
520 rhaas 482 UIC 0 : XLogFileName(startfname, state.starttli, startsegno,
483 : wal_segment_size);
3748 heikki.linnakangas 484 LBC 0 : ereport(ERROR,
3698 peter_e 485 EUB : (errmsg("could not find WAL file \"%s\"", startfname)));
486 : }
1363 tgl 487 GIC 40 : foreach(lc, walFileList)
488 : {
489 20 : char *walFileName = (char *) lfirst(lc);
3602 bruce 490 20 : XLogSegNo currsegno = segno;
491 20 : XLogSegNo nextsegno = segno + 1;
4452 magnus 492 ECB :
1363 tgl 493 GIC 20 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
3748 heikki.linnakangas 494 CBC 20 : if (!(nextsegno == segno || currsegno == segno))
495 : {
496 : char nextfname[MAXFNAMELEN];
497 :
527 rhaas 498 UBC 0 : XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
3748 heikki.linnakangas 499 UIC 0 : ereport(ERROR,
2118 tgl 500 EUB : (errmsg("could not find WAL file \"%s\"", nextfname)));
501 : }
502 : }
3748 heikki.linnakangas 503 CBC 20 : if (segno != endsegno)
504 : {
3602 bruce 505 ECB : char endfname[MAXFNAMELEN];
506 :
527 rhaas 507 LBC 0 : XLogFileName(endfname, endtli, endsegno, wal_segment_size);
3748 heikki.linnakangas 508 UIC 0 : ereport(ERROR,
3698 peter_e 509 ECB : (errmsg("could not find WAL file \"%s\"", endfname)));
3748 heikki.linnakangas 510 : }
511 :
512 : /* Ok, we have everything we need. Send the WAL files. */
1363 tgl 513 GIC 40 : foreach(lc, walFileList)
3748 heikki.linnakangas 514 EUB : {
1363 tgl 515 GBC 20 : char *walFileName = (char *) lfirst(lc);
516 : int fd;
517 : size_t cnt;
3748 heikki.linnakangas 518 GIC 20 : pgoff_t len = 0;
3748 heikki.linnakangas 519 ECB :
1363 tgl 520 GIC 20 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
521 20 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
522 :
1026 rhaas 523 GBC 20 : fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
524 20 : if (fd < 0)
525 : {
1749 michael 526 UIC 0 : int save_errno = errno;
527 :
528 : /*
3748 heikki.linnakangas 529 ECB : * Most likely reason for this is that the file was already
530 : * removed by a checkpoint, so check for that to get a better
531 : * error message.
532 : */
3748 heikki.linnakangas 533 UIC 0 : CheckXLogRemoved(segno, tli);
3748 heikki.linnakangas 534 ECB :
1749 michael 535 UIC 0 : errno = save_errno;
3748 heikki.linnakangas 536 LBC 0 : ereport(ERROR,
3748 heikki.linnakangas 537 ECB : (errcode_for_file_access(),
538 : errmsg("could not open file \"%s\": %m", pathbuf)));
539 : }
540 :
1026 rhaas 541 GIC 20 : if (fstat(fd, &statbuf) != 0)
3748 heikki.linnakangas 542 UBC 0 : ereport(ERROR,
543 : (errcode_for_file_access(),
544 : errmsg("could not stat file \"%s\": %m",
545 : pathbuf)));
2028 andres 546 GIC 20 : if (statbuf.st_size != wal_segment_size)
547 : {
3748 heikki.linnakangas 548 UIC 0 : CheckXLogRemoved(segno, tli);
3748 heikki.linnakangas 549 UBC 0 : ereport(ERROR,
550 : (errcode_for_file_access(),
1363 tgl 551 EUB : errmsg("unexpected WAL file size \"%s\"", walFileName)));
3748 heikki.linnakangas 552 : }
553 :
554 : /* send the WAL file itself */
520 rhaas 555 GIC 20 : _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
556 :
520 rhaas 557 CBC 20 : while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
520 rhaas 558 GBC 10240 : Min(sink->bbs_buffer_length,
559 : wal_segment_size - len),
1026 rhaas 560 GIC 10240 : len, pathbuf, true)) > 0)
561 : {
3748 heikki.linnakangas 562 CBC 10240 : CheckXLogRemoved(segno, tli);
520 rhaas 563 GIC 10240 : bbsink_archive_contents(sink, cnt);
3748 heikki.linnakangas 564 EUB :
3748 heikki.linnakangas 565 GBC 10240 : len += cnt;
566 :
2028 andres 567 GIC 10240 : if (len == wal_segment_size)
3748 heikki.linnakangas 568 20 : break;
569 : }
570 :
2028 andres 571 CBC 20 : if (len != wal_segment_size)
572 : {
3748 heikki.linnakangas 573 LBC 0 : CheckXLogRemoved(segno, tli);
574 0 : ereport(ERROR,
575 : (errcode_for_file_access(),
1363 tgl 576 ECB : errmsg("unexpected WAL file size \"%s\"", walFileName)));
577 : }
3748 heikki.linnakangas 578 :
1080 rhaas 579 : /*
580 : * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
581 : * for padding.
582 : */
1080 rhaas 583 CBC 20 : Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
3018 andres 584 ECB :
1026 rhaas 585 GIC 20 : CloseTransientFile(fd);
586 :
3018 andres 587 ECB : /*
588 : * Mark file as archived, otherwise files can get archived again
3018 andres 589 EUB : * after promotion of a new node. This is in line with
2881 heikki.linnakangas 590 : * walreceiver.c always doing an XLogArchiveForceDone() after a
591 : * complete segment.
592 : */
1363 tgl 593 GIC 20 : StatusFilePath(pathbuf, walFileName, ".done");
520 rhaas 594 20 : sendFileWithContent(sink, pathbuf, "", &manifest);
595 : }
596 :
597 : /*
598 : * Send timeline history files too. Only the latest timeline history
3748 heikki.linnakangas 599 ECB : * file is required for recovery, and even that only if there happens
600 : * to be a timeline switch in the first WAL segment that contains the
601 : * checkpoint record, or if we're taking a base backup from a standby
602 : * server and the target timeline changes while the backup is taken.
603 : * But they are small and highly useful for debugging purposes, so
604 : * better include them all, always.
605 : */
3748 heikki.linnakangas 606 GIC 20 : foreach(lc, historyFileList)
607 : {
3602 bruce 608 UIC 0 : char *fname = lfirst(lc);
3602 bruce 609 ECB :
3748 heikki.linnakangas 610 LBC 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
611 :
3748 heikki.linnakangas 612 UIC 0 : if (lstat(pathbuf, &statbuf) != 0)
613 0 : ereport(ERROR,
614 : (errcode_for_file_access(),
615 : errmsg("could not stat file \"%s\": %m", pathbuf)));
616 :
520 rhaas 617 0 : sendFile(sink, pathbuf, pathbuf, &statbuf, false, InvalidOid,
618 : &manifest, NULL);
619 :
620 : /* unconditionally mark file as archived */
3018 andres 621 0 : StatusFilePath(pathbuf, fname, ".done");
520 rhaas 622 LBC 0 : sendFileWithContent(sink, pathbuf, "", &manifest);
623 : }
4452 magnus 624 EUB :
625 : /* Properly terminate the tar file. */
515 rhaas 626 : StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
627 : "BLCKSZ too small for 2 tar blocks");
516 rhaas 628 GBC 20 : memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
629 20 : bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
630 :
631 : /* OK, that's the end of the archive. */
520 rhaas 632 GIC 20 : bbsink_end_archive(sink);
4452 magnus 633 EUB : }
634 :
520 rhaas 635 GIC 120 : AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
636 : endptr, endtli);
1101 rhaas 637 EUB :
520 rhaas 638 GBC 120 : SendBackupManifest(&manifest, sink);
639 :
520 rhaas 640 GIC 120 : bbsink_end_backup(sink, endptr, endtli);
641 :
1832 magnus 642 120 : if (total_checksum_failures)
643 : {
1832 magnus 644 CBC 3 : if (total_checksum_failures > 1)
645 2 : ereport(WARNING,
646 : (errmsg_plural("%lld total checksum verification failure",
647 : "%lld total checksum verification failures",
937 peter 648 ECB : total_checksum_failures,
649 : total_checksum_failures)));
650 :
1832 magnus 651 CBC 3 : ereport(ERROR,
652 : (errcode(ERRCODE_DATA_CORRUPTED),
653 : errmsg("checksum verification failure during base backup")));
1832 magnus 654 ECB : }
655 :
856 michael 656 : /*
657 : * Make sure to free the manifest before the resource owners as manifests
658 : * use cryptohash contexts that may depend on resource owners (like
659 : * OpenSSL).
660 : */
856 michael 661 CBC 117 : FreeBackupManifest(&manifest);
662 :
663 : /* clean up the resource owner we created */
1101 rhaas 664 GIC 117 : WalSndResourceCleanup(true);
665 :
520 666 117 : basebackup_progress_done();
4471 tgl 667 CBC 117 : }
668 :
669 : /*
670 : * list_sort comparison function, to compare log/seg portion of WAL segment
671 : * filenames, ignoring the timeline portion.
672 : */
673 : static int
1363 tgl 674 UIC 0 : compareWalFileNames(const ListCell *a, const ListCell *b)
675 : {
676 0 : char *fna = (char *) lfirst(a);
1363 tgl 677 LBC 0 : char *fnb = (char *) lfirst(b);
678 :
3748 heikki.linnakangas 679 UIC 0 : return strcmp(fna + 8, fnb + 8);
3748 heikki.linnakangas 680 ECB : }
681 :
4459 magnus 682 : /*
683 : * Parse the base backup options passed down by the parser
684 : */
685 : static void
4459 magnus 686 GIC 142 : parse_basebackup_options(List *options, basebackup_options *opt)
687 : {
688 : ListCell *lopt;
689 142 : bool o_label = false;
4459 magnus 690 GBC 142 : bool o_progress = false;
551 rhaas 691 GIC 142 : bool o_checkpoint = false;
4442 magnus 692 GBC 142 : bool o_nowait = false;
4452 693 142 : bool o_wal = false;
3328 alvherre 694 GIC 142 : bool o_maxrate = false;
2889 andrew 695 GBC 142 : bool o_tablespace_map = false;
1832 magnus 696 GIC 142 : bool o_noverify_checksums = false;
1101 rhaas 697 142 : bool o_manifest = false;
698 142 : bool o_manifest_checksums = false;
446 699 142 : bool o_target = false;
509 700 142 : bool o_target_detail = false;
390 701 142 : char *target_str = NULL;
390 rhaas 702 CBC 142 : char *target_detail_str = NULL;
440 rhaas 703 GIC 142 : bool o_compression = false;
382 704 142 : bool o_compression_detail = false;
382 rhaas 705 CBC 142 : char *compression_detail_str = NULL;
4459 magnus 706 ECB :
4457 magnus 707 CBC 1562 : MemSet(opt, 0, sizeof(*opt));
1101 rhaas 708 142 : opt->manifest = MANIFEST_OPTION_NO;
709 142 : opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
362 michael 710 142 : opt->compression = PG_COMPRESSION_NONE;
711 142 : opt->compression_specification.algorithm = PG_COMPRESSION_NONE;
1101 rhaas 712 ECB :
4459 magnus 713 CBC 1087 : foreach(lopt, options)
4459 magnus 714 ECB : {
4459 magnus 715 CBC 947 : DefElem *defel = (DefElem *) lfirst(lopt);
4459 magnus 716 ECB :
4459 magnus 717 CBC 947 : if (strcmp(defel->defname, "label") == 0)
4459 magnus 718 ECB : {
4459 magnus 719 CBC 142 : if (o_label)
4459 magnus 720 LBC 0 : ereport(ERROR,
4459 magnus 721 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
722 : errmsg("duplicate option \"%s\"", defel->defname)));
551 rhaas 723 CBC 142 : opt->label = defGetString(defel);
4459 magnus 724 142 : o_label = true;
4459 magnus 725 ECB : }
4459 magnus 726 CBC 805 : else if (strcmp(defel->defname, "progress") == 0)
4459 magnus 727 ECB : {
4459 magnus 728 GIC 142 : if (o_progress)
4459 magnus 729 LBC 0 : ereport(ERROR,
730 : (errcode(ERRCODE_SYNTAX_ERROR),
4459 magnus 731 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
551 rhaas 732 GIC 142 : opt->progress = defGetBoolean(defel);
4459 magnus 733 CBC 142 : o_progress = true;
734 : }
551 rhaas 735 663 : else if (strcmp(defel->defname, "checkpoint") == 0)
4459 magnus 736 EUB : {
551 rhaas 737 GIC 132 : char *optval = defGetString(defel);
738 :
551 rhaas 739 CBC 132 : if (o_checkpoint)
4459 magnus 740 LBC 0 : ereport(ERROR,
741 : (errcode(ERRCODE_SYNTAX_ERROR),
4459 magnus 742 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
551 rhaas 743 GIC 132 : if (pg_strcasecmp(optval, "fast") == 0)
551 rhaas 744 CBC 132 : opt->fastcheckpoint = true;
551 rhaas 745 UBC 0 : else if (pg_strcasecmp(optval, "spread") == 0)
551 rhaas 746 UIC 0 : opt->fastcheckpoint = false;
747 : else
551 rhaas 748 LBC 0 : ereport(ERROR,
551 rhaas 749 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
750 : errmsg("unrecognized checkpoint type: \"%s\"",
751 : optval)));
551 rhaas 752 GIC 132 : o_checkpoint = true;
4459 magnus 753 ECB : }
551 rhaas 754 GIC 531 : else if (strcmp(defel->defname, "wait") == 0)
4442 magnus 755 ECB : {
4442 magnus 756 GBC 135 : if (o_nowait)
4442 magnus 757 UIC 0 : ereport(ERROR,
758 : (errcode(ERRCODE_SYNTAX_ERROR),
4442 magnus 759 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
551 rhaas 760 CBC 135 : opt->nowait = !defGetBoolean(defel);
4442 magnus 761 GBC 135 : o_nowait = true;
4442 magnus 762 EUB : }
4452 magnus 763 GIC 396 : else if (strcmp(defel->defname, "wal") == 0)
4452 magnus 764 EUB : {
4452 magnus 765 GIC 24 : if (o_wal)
4452 magnus 766 UIC 0 : ereport(ERROR,
767 : (errcode(ERRCODE_SYNTAX_ERROR),
4452 magnus 768 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
551 rhaas 769 GIC 24 : opt->includewal = defGetBoolean(defel);
4452 magnus 770 CBC 24 : o_wal = true;
771 : }
3328 alvherre 772 372 : else if (strcmp(defel->defname, "max_rate") == 0)
3328 alvherre 773 EUB : {
774 : int64 maxrate;
775 :
3328 alvherre 776 CBC 1 : if (o_maxrate)
3328 alvherre 777 LBC 0 : ereport(ERROR,
778 : (errcode(ERRCODE_SYNTAX_ERROR),
3328 alvherre 779 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
780 :
551 rhaas 781 CBC 1 : maxrate = defGetInt64(defel);
3328 alvherre 782 GBC 1 : if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
3328 alvherre 783 UIC 0 : ereport(ERROR,
784 : (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
3328 alvherre 785 ECB : errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
2118 tgl 786 : (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
787 :
3328 alvherre 788 CBC 1 : opt->maxrate = (uint32) maxrate;
3328 alvherre 789 GIC 1 : o_maxrate = true;
790 : }
2889 andrew 791 371 : else if (strcmp(defel->defname, "tablespace_map") == 0)
2889 andrew 792 ECB : {
2889 andrew 793 GBC 29 : if (o_tablespace_map)
2889 andrew 794 UIC 0 : ereport(ERROR,
795 : (errcode(ERRCODE_SYNTAX_ERROR),
796 : errmsg("duplicate option \"%s\"", defel->defname)));
551 rhaas 797 CBC 29 : opt->sendtblspcmapfile = defGetBoolean(defel);
2889 andrew 798 29 : o_tablespace_map = true;
2889 andrew 799 EUB : }
551 rhaas 800 GIC 342 : else if (strcmp(defel->defname, "verify_checksums") == 0)
801 : {
1832 magnus 802 1 : if (o_noverify_checksums)
1832 magnus 803 UIC 0 : ereport(ERROR,
1832 magnus 804 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
805 : errmsg("duplicate option \"%s\"", defel->defname)));
551 rhaas 806 GIC 1 : noverify_checksums = !defGetBoolean(defel);
1832 magnus 807 CBC 1 : o_noverify_checksums = true;
808 : }
1101 rhaas 809 341 : else if (strcmp(defel->defname, "manifest") == 0)
1101 rhaas 810 EUB : {
551 rhaas 811 GIC 141 : char *optval = defGetString(defel);
812 : bool manifest_bool;
1101 rhaas 813 ECB :
1101 rhaas 814 CBC 141 : if (o_manifest)
1101 rhaas 815 UIC 0 : ereport(ERROR,
1101 rhaas 816 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
817 : errmsg("duplicate option \"%s\"", defel->defname)));
1101 rhaas 818 CBC 141 : if (parse_bool(optval, &manifest_bool))
1101 rhaas 819 EUB : {
1101 rhaas 820 GIC 140 : if (manifest_bool)
821 140 : opt->manifest = MANIFEST_OPTION_YES;
1101 rhaas 822 ECB : else
1101 rhaas 823 LBC 0 : opt->manifest = MANIFEST_OPTION_NO;
824 : }
1101 rhaas 825 CBC 1 : else if (pg_strcasecmp(optval, "force-encode") == 0)
1101 rhaas 826 GIC 1 : opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
1101 rhaas 827 ECB : else
1101 rhaas 828 UIC 0 : ereport(ERROR,
829 : (errcode(ERRCODE_SYNTAX_ERROR),
1101 rhaas 830 ECB : errmsg("unrecognized manifest option: \"%s\"",
1101 rhaas 831 EUB : optval)));
1101 rhaas 832 GIC 141 : o_manifest = true;
833 : }
1101 rhaas 834 CBC 200 : else if (strcmp(defel->defname, "manifest_checksums") == 0)
835 : {
551 836 7 : char *optval = defGetString(defel);
1101 rhaas 837 ECB :
1101 rhaas 838 GIC 7 : if (o_manifest_checksums)
1101 rhaas 839 UBC 0 : ereport(ERROR,
840 : (errcode(ERRCODE_SYNTAX_ERROR),
1101 rhaas 841 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
1101 rhaas 842 CBC 7 : if (!pg_checksum_parse_type(optval,
843 : &opt->manifest_checksum_type))
1101 rhaas 844 GBC 1 : ereport(ERROR,
845 : (errcode(ERRCODE_SYNTAX_ERROR),
846 : errmsg("unrecognized checksum algorithm: \"%s\"",
847 : optval)));
1101 rhaas 848 CBC 6 : o_manifest_checksums = true;
849 : }
446 850 193 : else if (strcmp(defel->defname, "target") == 0)
851 : {
852 141 : if (o_target)
446 rhaas 853 UIC 0 : ereport(ERROR,
446 rhaas 854 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
446 rhaas 855 EUB : errmsg("duplicate option \"%s\"", defel->defname)));
390 rhaas 856 GIC 141 : target_str = defGetString(defel);
446 857 141 : o_target = true;
446 rhaas 858 ECB : }
509 rhaas 859 GIC 52 : else if (strcmp(defel->defname, "target_detail") == 0)
509 rhaas 860 ECB : {
509 rhaas 861 GIC 9 : char *optval = defGetString(defel);
862 :
863 9 : if (o_target_detail)
509 rhaas 864 LBC 0 : ereport(ERROR,
865 : (errcode(ERRCODE_SYNTAX_ERROR),
509 rhaas 866 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
390 rhaas 867 GIC 9 : target_detail_str = optval;
509 rhaas 868 CBC 9 : o_target_detail = true;
509 rhaas 869 EUB : }
440 rhaas 870 GIC 43 : else if (strcmp(defel->defname, "compression") == 0)
871 : {
440 rhaas 872 CBC 29 : char *optval = defGetString(defel);
440 rhaas 873 ECB :
440 rhaas 874 GIC 29 : if (o_compression)
440 rhaas 875 LBC 0 : ereport(ERROR,
876 : (errcode(ERRCODE_SYNTAX_ERROR),
440 rhaas 877 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
362 michael 878 GIC 29 : if (!parse_compress_algorithm(optval, &opt->compression))
440 rhaas 879 CBC 1 : ereport(ERROR,
440 rhaas 880 EUB : (errcode(ERRCODE_SYNTAX_ERROR),
881 : errmsg("unrecognized compression algorithm: \"%s\"",
882 : optval)));
440 rhaas 883 CBC 28 : o_compression = true;
440 rhaas 884 ECB : }
382 rhaas 885 GIC 14 : else if (strcmp(defel->defname, "compression_detail") == 0)
440 rhaas 886 ECB : {
382 rhaas 887 GIC 14 : if (o_compression_detail)
440 rhaas 888 LBC 0 : ereport(ERROR,
889 : (errcode(ERRCODE_SYNTAX_ERROR),
440 rhaas 890 ECB : errmsg("duplicate option \"%s\"", defel->defname)));
382 rhaas 891 GBC 14 : compression_detail_str = defGetString(defel);
382 rhaas 892 GIC 14 : o_compression_detail = true;
893 : }
440 rhaas 894 ECB : else
440 rhaas 895 LBC 0 : ereport(ERROR,
896 : (errcode(ERRCODE_SYNTAX_ERROR),
897 : errmsg("unrecognized base backup option: \"%s\"",
898 : defel->defname)));
4459 magnus 899 ECB : }
900 :
4459 magnus 901 CBC 140 : if (opt->label == NULL)
4459 magnus 902 UIC 0 : opt->label = "base backup";
1101 rhaas 903 CBC 140 : if (opt->manifest == MANIFEST_OPTION_NO)
1101 rhaas 904 EUB : {
1101 rhaas 905 GIC 1 : if (o_manifest_checksums)
1101 rhaas 906 UIC 0 : ereport(ERROR,
1101 rhaas 907 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
908 : errmsg("manifest checksums require a backup manifest")));
1101 rhaas 909 GIC 1 : opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
910 : }
390 rhaas 911 EUB :
390 rhaas 912 GIC 140 : if (target_str == NULL)
913 : {
390 rhaas 914 UIC 0 : if (target_detail_str != NULL)
509 915 0 : ereport(ERROR,
916 : (errcode(ERRCODE_SYNTAX_ERROR),
390 rhaas 917 ECB : errmsg("target detail cannot be used without target")));
390 rhaas 918 UBC 0 : opt->use_copytblspc = true;
390 rhaas 919 LBC 0 : opt->send_to_client = true;
920 : }
390 rhaas 921 CBC 140 : else if (strcmp(target_str, "client") == 0)
509 rhaas 922 EUB : {
390 rhaas 923 GIC 125 : if (target_detail_str != NULL)
509 rhaas 924 UIC 0 : ereport(ERROR,
509 rhaas 925 ECB : (errcode(ERRCODE_SYNTAX_ERROR),
926 : errmsg("target \"%s\" does not accept a target detail",
927 : target_str)));
390 rhaas 928 CBC 125 : opt->send_to_client = true;
929 : }
390 rhaas 930 EUB : else
390 rhaas 931 GBC 13 : opt->target_handle =
390 rhaas 932 GIC 15 : BaseBackupGetTargetHandle(target_str, target_detail_str);
933 :
382 rhaas 934 GBC 138 : if (o_compression_detail && !o_compression)
440 rhaas 935 UBC 0 : ereport(ERROR,
936 : (errcode(ERRCODE_SYNTAX_ERROR),
197 peter 937 ECB : errmsg("compression detail cannot be specified unless compression is enabled")));
938 :
382 rhaas 939 CBC 138 : if (o_compression)
382 rhaas 940 EUB : {
941 : char *error_detail;
942 :
362 michael 943 GIC 26 : parse_compress_specification(opt->compression, compression_detail_str,
362 michael 944 ECB : &opt->compression_specification);
945 : error_detail =
362 michael 946 GIC 26 : validate_compress_specification(&opt->compression_specification);
382 rhaas 947 CBC 26 : if (error_detail != NULL)
948 9 : ereport(ERROR,
949 : errcode(ERRCODE_SYNTAX_ERROR),
382 rhaas 950 ECB : errmsg("invalid compression specification: %s",
382 rhaas 951 EUB : error_detail));
952 : }
4459 magnus 953 GIC 129 : }
954 :
4459 magnus 955 ECB :
956 : /*
957 : * SendBaseBackup() - send a complete base backup.
958 : *
368 sfrost 959 : * The function will put the system into backup mode like pg_backup_start()
960 : * does, so that the backup is consistent even though we read directly from
961 : * the filesystem, bypassing the buffer cache.
4472 magnus 962 : */
963 : void
4459 magnus 964 CBC 143 : SendBaseBackup(BaseBackupCmd *cmd)
965 : {
966 : basebackup_options opt;
967 : bbsink *sink;
271 fujii 968 GIC 143 : SessionBackupState status = get_backup_status();
271 fujii 969 ECB :
271 fujii 970 GIC 143 : if (status == SESSION_BACKUP_RUNNING)
971 1 : ereport(ERROR,
972 : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
973 : errmsg("a backup is already in progress in this session")));
974 :
4459 magnus 975 142 : parse_basebackup_options(cmd->options, &opt);
976 :
4471 977 129 : WalSndSetState(WALSNDSTATE_BACKUP);
978 :
4472 979 129 : if (update_process_title)
4472 magnus 980 ECB : {
981 : char activitymsg[50];
982 :
4472 magnus 983 GIC 129 : snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
4459 magnus 984 ECB : opt.label);
1124 peter 985 GIC 129 : set_ps_display(activitymsg);
4472 magnus 986 ECB : }
987 :
988 : /*
989 : * If the target is specifically 'client' then set up to stream the backup
990 : * to the client; otherwise, it's being sent someplace else and should not
390 rhaas 991 : * be sent to the client. BaseBackupGetSink has the job of setting up a
992 : * sink to send the backup data wherever it needs to go.
446 993 : */
390 rhaas 994 GIC 129 : sink = bbsink_copystream_new(opt.send_to_client);
390 rhaas 995 CBC 129 : if (opt.target_handle != NULL)
390 rhaas 996 GIC 13 : sink = BaseBackupGetSink(opt.target_handle, sink);
997 :
998 : /* Set up network throttling, if client requested it */
520 rhaas 999 CBC 126 : if (opt.maxrate > 0)
520 rhaas 1000 GIC 1 : sink = bbsink_throttle_new(sink, opt.maxrate);
4472 magnus 1001 ECB :
1002 : /* Set up server-side compression, if client requested it */
362 michael 1003 GIC 126 : if (opt.compression == PG_COMPRESSION_GZIP)
382 rhaas 1004 2 : sink = bbsink_gzip_new(sink, &opt.compression_specification);
362 michael 1005 124 : else if (opt.compression == PG_COMPRESSION_LZ4)
382 rhaas 1006 2 : sink = bbsink_lz4_new(sink, &opt.compression_specification);
362 michael 1007 122 : else if (opt.compression == PG_COMPRESSION_ZSTD)
382 rhaas 1008 4 : sink = bbsink_zstd_new(sink, &opt.compression_specification);
1009 :
520 rhaas 1010 ECB : /* Set up progress reporting. */
520 rhaas 1011 CBC 126 : sink = bbsink_progress_new(sink, opt.progress);
3602 bruce 1012 ECB :
1013 : /*
1014 : * Perform the base backup, but make sure we clean up the bbsink even if
520 rhaas 1015 : * an error occurs.
3734 heikki.linnakangas 1016 : */
520 rhaas 1017 GIC 126 : PG_TRY();
1018 : {
520 rhaas 1019 CBC 126 : perform_base_backup(&opt, sink);
520 rhaas 1020 ECB : }
520 rhaas 1021 CBC 4 : PG_FINALLY();
520 rhaas 1022 ECB : {
520 rhaas 1023 CBC 121 : bbsink_cleanup(sink);
520 rhaas 1024 ECB : }
520 rhaas 1025 GIC 121 : PG_END_TRY();
4448 magnus 1026 117 : }
4448 magnus 1027 ECB :
1028 : /*
1029 : * Inject a file with given name and content in the output tar stream.
1030 : */
1031 : static void
520 rhaas 1032 GIC 170 : sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1081 rhaas 1033 ECB : backup_manifest_info *manifest)
1034 : {
4451 heikki.linnakangas 1035 : struct stat statbuf;
520 rhaas 1036 GIC 170 : int bytes_done = 0,
4382 bruce 1037 ECB : len;
1038 : pg_checksum_context checksum_ctx;
1101 rhaas 1039 :
858 michael 1040 GIC 170 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
858 michael 1041 LBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
858 michael 1042 ECB : filename);
1043 :
4451 heikki.linnakangas 1044 GIC 170 : len = strlen(content);
1045 :
1046 : /*
1047 : * Construct a stat struct for the backup_label file we're injecting in
4382 bruce 1048 ECB : * the tar.
1049 : */
1050 : /* Windows doesn't have the concept of uid and gid */
1051 : #ifdef WIN32
4451 heikki.linnakangas 1052 : statbuf.st_uid = 0;
1053 : statbuf.st_gid = 0;
1054 : #else
4451 heikki.linnakangas 1055 GIC 170 : statbuf.st_uid = geteuid();
4451 heikki.linnakangas 1056 CBC 170 : statbuf.st_gid = getegid();
4451 heikki.linnakangas 1057 EUB : #endif
4451 heikki.linnakangas 1058 GIC 170 : statbuf.st_mtime = time(NULL);
1828 sfrost 1059 170 : statbuf.st_mode = pg_file_create_mode;
4451 heikki.linnakangas 1060 CBC 170 : statbuf.st_size = len;
1061 :
520 rhaas 1062 GIC 170 : _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1063 :
1064 170 : if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
520 rhaas 1065 UIC 0 : elog(ERROR, "could not update checksum of file \"%s\"",
1066 : filename);
1067 :
520 rhaas 1068 GIC 298 : while (bytes_done < len)
1069 : {
1070 128 : size_t remaining = len - bytes_done;
520 rhaas 1071 CBC 128 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
4382 bruce 1072 ECB :
520 rhaas 1073 GIC 128 : memcpy(sink->bbs_buffer, content, nbytes);
520 rhaas 1074 CBC 128 : bbsink_archive_contents(sink, nbytes);
1075 128 : bytes_done += nbytes;
118 1076 128 : content += nbytes;
1077 : }
1101 rhaas 1078 ECB :
520 rhaas 1079 GIC 170 : _tarWritePadding(sink, len);
858 michael 1080 ECB :
1081 rhaas 1081 GBC 170 : AddFileToBackupManifest(manifest, NULL, filename, len,
1081 rhaas 1082 GIC 170 : (pg_time_t) statbuf.st_mtime, &checksum_ctx);
4451 heikki.linnakangas 1083 170 : }
4472 magnus 1084 ECB :
1085 : /*
3667 heikki.linnakangas 1086 : * Include the tablespace directory pointed to by 'path' in the output tar
3260 bruce 1087 : * stream. If 'sizeonly' is true, we just calculate a total length and return
1088 : * it, without actually sending anything.
3379 magnus 1089 : *
1090 : * Only used to send auxiliary tablespaces, not PGDATA.
3667 heikki.linnakangas 1091 : */
1026 rhaas 1092 : static int64
520 rhaas 1093 GIC 32 : sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly,
1094 : backup_manifest_info *manifest)
3667 heikki.linnakangas 1095 ECB : {
1096 : int64 size;
1097 : char pathbuf[MAXPGPATH];
1098 : struct stat statbuf;
1099 :
1100 : /*
1101 : * 'path' points to the tablespace location, but we only want to include
1102 : * the version directory in it that belongs to us.
1103 : */
3667 heikki.linnakangas 1104 GIC 32 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1105 : TABLESPACE_VERSION_DIRECTORY);
1106 :
1107 : /*
1108 : * Store a directory entry in the tar file so we get the permissions
3602 bruce 1109 ECB : * right.
1110 : */
3667 heikki.linnakangas 1111 GIC 32 : if (lstat(pathbuf, &statbuf) != 0)
1112 : {
3667 heikki.linnakangas 1113 UIC 0 : if (errno != ENOENT)
1114 0 : ereport(ERROR,
1115 : (errcode_for_file_access(),
1116 : errmsg("could not stat file or directory \"%s\": %m",
1117 : pathbuf)));
1118 :
1119 : /* If the tablespace went away while scanning, it's no error. */
3667 heikki.linnakangas 1120 LBC 0 : return 0;
1121 : }
1122 :
520 rhaas 1123 GIC 32 : size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1124 : sizeonly);
1125 :
1126 : /* Send all the files in the tablespace version directory */
520 rhaas 1127 CBC 32 : size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1128 : spcoid);
3667 heikki.linnakangas 1129 EUB :
3667 heikki.linnakangas 1130 GBC 32 : return size;
1131 : }
1132 :
1133 : /*
1134 : * Include all files from the given directory in the output tar stream. If
1135 : * 'sizeonly' is true, we just calculate a total length and return it, without
4451 heikki.linnakangas 1136 EUB : * actually sending anything.
1137 : *
1138 : * Omit any directory in the tablespaces list, to avoid backing up
3379 magnus 1139 ECB : * tablespaces twice when they were created inside PGDATA.
1140 : *
1141 : * If sendtblspclinks is true, we need to include symlink
1142 : * information in the tar file. If not, we can skip that
2889 andrew 1143 : * as it will be sent separately in the tablespace_map file.
1144 : */
1145 : static int64
520 rhaas 1146 CBC 4316 : sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1147 : List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1148 : const char *spcoid)
1149 : {
1150 : DIR *dir;
1151 : struct dirent *de;
1152 : char pathbuf[MAXPGPATH * 2];
1153 : struct stat statbuf;
4472 magnus 1154 GIC 4316 : int64 size = 0;
1155 : const char *lastDir; /* Split last dir from parent path. */
1809 tgl 1156 4316 : bool isDbDir = false; /* Does this directory contain relations? */
1157 :
1158 : /*
1159 : * Determine if the current path is a database directory that can contain
1160 : * relations.
1161 : *
1809 tgl 1162 ECB : * Start by finding the location of the delimiter between the parent path
1163 : * and the current path.
1164 : */
1843 teodor 1165 GIC 4316 : lastDir = last_dir_separator(path);
1166 :
1167 : /* Does this path look like a database path (i.e. all digits)? */
1168 4316 : if (lastDir != NULL &&
1169 4064 : strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1843 teodor 1170 ECB : {
1171 : /* Part of path that contains the parent directory. */
1809 tgl 1172 CBC 811 : int parentPathLen = lastDir - path;
1173 :
1174 : /*
1175 : * Mark path as a database directory if the parent path is either
1176 : * $PGDATA/base or a tablespace version path.
1177 : */
1843 teodor 1178 GIC 811 : if (strncmp(path, "./base", parentPathLen) == 0 ||
1179 54 : (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1180 32 : strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1843 teodor 1181 ECB : TABLESPACE_VERSION_DIRECTORY,
1182 : sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1843 teodor 1183 GIC 789 : isDbDir = true;
1843 teodor 1184 ECB : }
4472 magnus 1185 :
4472 magnus 1186 GIC 4316 : dir = AllocateDir(path);
1187 263588 : while ((de = ReadDir(dir, path)) != NULL)
4472 magnus 1188 ECB : {
1189 : int excludeIdx;
1190 : bool excludeFound;
1191 : ForkNumber relForkNum; /* Type of fork if file is a relation */
1192 : int relnumchars; /* Chars in filename that are the
1193 : * relnumber */
1194 :
1195 : /* Skip special stuff */
4472 magnus 1196 CBC 259286 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1197 12211 : continue;
1198 :
1199 : /* Skip temporary files */
heikki.linnakangas 1200 250667 : if (strncmp(de->d_name,
1201 : PG_TEMP_FILE_PREFIX,
1202 : strlen(PG_TEMP_FILE_PREFIX)) == 0)
1203 247 : continue;
4472 heikki.linnakangas 1204 ECB :
1205 : /*
1206 : * Check if the postmaster has signaled us to exit, and abort with an
1207 : * error in that case. The error handler further up will call
1208 : * do_pg_abort_backup() for us. Also check that if the backup was
1209 : * started while still in recovery, the server wasn't promoted.
1210 : * do_pg_backup_stop() will check that too, but it's better to stop
1211 : * the backup early than continue to the end and fail there.
1212 : */
3769 heikki.linnakangas 1213 CBC 250420 : CHECK_FOR_INTERRUPTS();
1214 250415 : if (RecoveryInProgress() != backup_started_in_recovery)
4468 magnus 1215 UIC 0 : ereport(ERROR,
1216 : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
3769 heikki.linnakangas 1217 ECB : errmsg("the standby was promoted during online backup"),
1218 : errhint("This means that the backup being taken is corrupt "
1219 : "and should not be used. "
2118 tgl 1220 : "Try taking another online backup.")));
1221 :
1222 : /* Scan for files that should be excluded */
2384 peter_e 1223 GIC 250415 : excludeFound = false;
1140 michael 1224 2249800 : for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1225 : {
1226 2000384 : int cmplen = strlen(excludeFiles[excludeIdx].name);
1227 :
1228 2000384 : if (!excludeFiles[excludeIdx].match_prefix)
1229 1750103 : cmplen++;
1140 michael 1230 CBC 2000384 : if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
2384 peter_e 1231 ECB : {
2384 peter_e 1232 GBC 999 : elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
2384 peter_e 1233 GIC 999 : excludeFound = true;
1234 999 : break;
1235 : }
1236 : }
1237 :
1238 250415 : if (excludeFound)
4472 magnus 1239 999 : continue;
4472 magnus 1240 ECB :
1843 teodor 1241 : /* Exclude all forks for unlogged tables except the init fork */
1843 teodor 1242 GIC 475116 : if (isDbDir &&
255 rhaas 1243 GNC 225700 : parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
1244 : &relForkNum))
1843 teodor 1245 ECB : {
1246 : /* Never exclude init forks */
1843 teodor 1247 CBC 224148 : if (relForkNum != INIT_FORKNUM)
1248 : {
1809 tgl 1249 ECB : char initForkFile[MAXPGPATH];
1250 : char relNumber[OIDCHARS + 1];
1843 teodor 1251 :
1252 : /*
1253 : * If any other type of fork, check if there is an init fork
1254 : * with the same RelFileNumber. If so, the file can be
1255 : * excluded.
1256 : */
255 rhaas 1257 GNC 224079 : memcpy(relNumber, de->d_name, relnumchars);
1258 224079 : relNumber[relnumchars] = '\0';
1843 teodor 1259 GIC 224079 : snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1260 : path, relNumber);
1843 teodor 1261 ECB :
1843 teodor 1262 GIC 224079 : if (lstat(initForkFile, &statbuf) == 0)
1263 : {
1264 69 : elog(DEBUG2,
1843 teodor 1265 ECB : "unlogged relation file \"%s\" excluded from backup",
1266 : de->d_name);
1267 :
1843 teodor 1268 GIC 69 : continue;
1269 : }
1270 : }
1271 : }
1272 :
1273 : /* Exclude temporary relations */
1839 1274 249347 : if (isDbDir && looks_like_temp_rel_name(de->d_name))
1839 teodor 1275 ECB : {
1839 teodor 1276 CBC 40 : elog(DEBUG2,
1839 teodor 1277 ECB : "temporary relation file \"%s\" excluded from backup",
1278 : de->d_name);
1279 :
1839 teodor 1280 CBC 40 : continue;
1281 : }
1839 teodor 1282 ECB :
2189 peter_e 1283 GIC 249307 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1284 :
1285 : /* Skip pg_control here to back up it last */
4092 simon 1286 CBC 249307 : if (strcmp(pathbuf, "./global/pg_control") == 0)
4092 simon 1287 GIC 247 : continue;
1288 :
4472 magnus 1289 249060 : if (lstat(pathbuf, &statbuf) != 0)
1290 : {
4472 magnus 1291 UIC 0 : if (errno != ENOENT)
4472 magnus 1292 LBC 0 : ereport(ERROR,
1293 : (errcode_for_file_access(),
4472 magnus 1294 ECB : errmsg("could not stat file or directory \"%s\": %m",
1295 : pathbuf)));
1296 :
1297 : /* If the file went away while scanning, it's not an error. */
4472 magnus 1298 LBC 0 : continue;
1299 : }
1300 :
2384 peter_e 1301 ECB : /* Scan for directories whose contents should be excluded */
2384 peter_e 1302 GIC 249060 : excludeFound = false;
1303 1985521 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
3352 fujii 1304 ECB : {
2384 peter_e 1305 CBC 1738199 : if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1306 : {
1307 1738 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
544 rhaas 1308 GIC 1738 : convert_link_to_directory(pathbuf, &statbuf);
520 rhaas 1309 GBC 1738 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
520 rhaas 1310 EUB : &statbuf, sizeonly);
2384 peter_e 1311 GIC 1738 : excludeFound = true;
1312 1738 : break;
1313 : }
1314 : }
1315 :
2384 peter_e 1316 GBC 249060 : if (excludeFound)
2384 peter_e 1317 GIC 1738 : continue;
1318 :
1319 : /*
2362 rhaas 1320 ECB : * We can skip pg_wal, the WAL segments need to be fetched from the
4472 magnus 1321 : * WAL archive anyway. But include it as an empty directory anyway, so
1322 : * we get permissions right.
1323 : */
2362 rhaas 1324 GIC 247322 : if (strcmp(pathbuf, "./pg_wal") == 0)
4472 magnus 1325 ECB : {
2362 rhaas 1326 : /* If pg_wal is a symlink, write it as a directory anyway */
544 rhaas 1327 CBC 252 : convert_link_to_directory(pathbuf, &statbuf);
520 rhaas 1328 GIC 252 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
520 rhaas 1329 ECB : &statbuf, sizeonly);
3018 andres 1330 :
1331 : /*
1332 : * Also send archive_status directory (by hackishly reusing
1333 : * statbuf from above ...).
1334 : */
520 rhaas 1335 CBC 252 : size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1336 : &statbuf, sizeonly);
1337 :
2362 rhaas 1338 GIC 252 : continue; /* don't recurse into pg_wal */
1339 : }
1340 :
1341 : /* Allow symbolic links in pg_tblspc only */
246 tmunro 1342 GNC 247070 : if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
4472 magnus 1343 GIC 29 : {
1344 : char linkpath[MAXPGPATH];
1345 : int rllen;
4472 magnus 1346 ECB :
4141 tgl 1347 GIC 29 : rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1348 29 : if (rllen < 0)
4472 magnus 1349 LBC 0 : ereport(ERROR,
1350 : (errcode_for_file_access(),
1351 : errmsg("could not read symbolic link \"%s\": %m",
1352 : pathbuf)));
4141 tgl 1353 CBC 29 : if (rllen >= sizeof(linkpath))
4141 tgl 1354 LBC 0 : ereport(ERROR,
1355 : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1356 : errmsg("symbolic link \"%s\" target is too long",
1357 : pathbuf)));
4141 tgl 1358 CBC 29 : linkpath[rllen] = '\0';
4141 tgl 1359 ECB :
520 rhaas 1360 GBC 29 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1361 : &statbuf, sizeonly);
4472 magnus 1362 ECB : }
4472 magnus 1363 GIC 247041 : else if (S_ISDIR(statbuf.st_mode))
4472 magnus 1364 ECB : {
3379 magnus 1365 GIC 4056 : bool skip_this_dir = false;
1366 : ListCell *lc;
1367 :
1368 : /*
1369 : * Store a directory entry in the tar file so we can get the
1370 : * permissions right.
4472 magnus 1371 ECB : */
520 rhaas 1372 GIC 4056 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1373 : sizeonly);
1374 :
1375 : /*
1376 : * Call ourselves recursively for a directory, unless it happens
1377 : * to be a separate tablespace located within PGDATA.
3379 magnus 1378 ECB : */
3379 magnus 1379 GIC 8580 : foreach(lc, tablespaces)
3379 magnus 1380 ECB : {
3379 magnus 1381 GIC 4524 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1382 :
1383 : /*
1384 : * ti->rpath is the tablespace relative path within PGDATA, or
1385 : * NULL if the tablespace has been properly located somewhere
1386 : * else.
1387 : *
1388 : * Skip past the leading "./" in pathbuf when comparing.
3379 magnus 1389 ECB : */
3379 magnus 1390 GIC 4524 : if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
3379 magnus 1391 EUB : {
3379 magnus 1392 UBC 0 : skip_this_dir = true;
3379 magnus 1393 UIC 0 : break;
1394 : }
1395 : }
1396 :
1397 : /*
1398 : * skip sending directories inside pg_tblspc, if not required.
2889 andrew 1399 ECB : */
2889 andrew 1400 CBC 4056 : if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
2889 andrew 1401 GIC 24 : skip_this_dir = true;
2889 andrew 1402 ECB :
3379 magnus 1403 CBC 4056 : if (!skip_this_dir)
520 rhaas 1404 GIC 4032 : size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1405 : sendtblspclinks, manifest, spcoid);
4472 magnus 1406 ECB : }
4472 magnus 1407 GIC 242985 : else if (S_ISREG(statbuf.st_mode))
4472 magnus 1408 ECB : {
3602 bruce 1409 GIC 242985 : bool sent = false;
3761 heikki.linnakangas 1410 ECB :
4472 magnus 1411 CBC 242985 : if (!sizeonly)
520 rhaas 1412 229966 : sent = sendFile(sink, pathbuf, pathbuf + basepathlen + 1, &statbuf,
1101 rhaas 1413 GIC 110721 : true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1414 : manifest, spcoid);
3761 heikki.linnakangas 1415 ECB :
3761 heikki.linnakangas 1416 GIC 242984 : if (sent || sizeonly)
1417 : {
1080 rhaas 1418 ECB : /* Add size. */
1080 rhaas 1419 GIC 242984 : size += statbuf.st_size;
1420 :
1080 rhaas 1421 ECB : /* Pad to a multiple of the tar block size. */
1080 rhaas 1422 GIC 242984 : size += tarPaddingBytesRequired(statbuf.st_size);
1423 :
1080 rhaas 1424 ECB : /* Size of the header for the file. */
1080 rhaas 1425 GIC 242984 : size += TAR_BLOCK_SIZE;
1426 : }
1427 : }
4472 magnus 1428 EUB : else
4472 magnus 1429 UIC 0 : ereport(WARNING,
1430 : (errmsg("skipping special file \"%s\"", pathbuf)));
4472 magnus 1431 ECB : }
4472 magnus 1432 CBC 4302 : FreeDir(dir);
4472 magnus 1433 GIC 4302 : return size;
1434 : }
1435 :
1436 : /*
1437 : * Check if a file should have its checksum validated.
1438 : * We validate checksums on files in regular tablespaces
1439 : * (including global and default) only, and in those there
1440 : * are some files that are explicitly excluded.
1441 : */
1832 magnus 1442 ECB : static bool
1832 magnus 1443 GIC 28387 : is_checksummed_file(const char *fullpath, const char *filename)
1444 : {
1832 magnus 1445 ECB : /* Check that the file is in a tablespace */
1832 magnus 1446 CBC 28387 : if (strncmp(fullpath, "./global/", 9) == 0 ||
1447 26618 : strncmp(fullpath, "./base/", 7) == 0 ||
1832 magnus 1448 GIC 329 : strncmp(fullpath, "/", 1) == 0)
1449 : {
1450 : int excludeIdx;
1451 :
1140 michael 1452 ECB : /* Compare file against noChecksumFiles skip list */
1140 michael 1453 GIC 139874 : for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1140 michael 1454 ECB : {
1140 michael 1455 GIC 112016 : int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1140 michael 1456 ECB :
1140 michael 1457 CBC 112016 : if (!noChecksumFiles[excludeIdx].match_prefix)
1458 84071 : cmplen++;
1140 michael 1459 GIC 112016 : if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1140 michael 1460 ECB : cmplen) == 0)
1832 magnus 1461 GIC 205 : return false;
1462 : }
1832 magnus 1463 ECB :
1832 magnus 1464 GIC 27858 : return true;
1465 : }
1832 magnus 1466 ECB : else
1832 magnus 1467 GIC 324 : return false;
1468 : }
1469 :
1470 : /*
1471 : * Given the member, write the TAR header & send the file.
1472 : *
1473 : * If 'missing_ok' is true, will not throw an error if the file is not found.
3761 heikki.linnakangas 1474 ECB : *
1475 : * If dboid is anything other than InvalidOid then any checksum failures
1476 : * detected will get reported to the cumulative stats system.
1477 : *
1478 : * Returns true if the file was successfully sent, false if 'missing_ok',
1479 : * and the file did not exist.
1480 : */
1481 : static bool
520 rhaas 1482 CBC 119365 : sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1483 : struct stat *statbuf, bool missing_ok, Oid dboid,
1484 : backup_manifest_info *manifest, const char *spcoid)
4472 magnus 1485 ECB : {
1486 : int fd;
1832 magnus 1487 GIC 119365 : BlockNumber blkno = 0;
1832 magnus 1488 CBC 119365 : bool block_retry = false;
1489 : uint16 checksum;
1490 119365 : int checksum_failures = 0;
1491 : off_t cnt;
1492 : int i;
4472 1493 119365 : pgoff_t len = 0;
1832 magnus 1494 EUB : char *page;
1495 : PageHeader phdr;
1832 magnus 1496 GIC 119365 : int segmentno = 0;
1832 magnus 1497 ECB : char *segmentpath;
1832 magnus 1498 CBC 119365 : bool verify_checksum = false;
1499 : pg_checksum_context checksum_ctx;
1101 rhaas 1500 EUB :
858 michael 1501 GBC 119365 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
858 michael 1502 UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1503 : readfilename);
1504 :
1026 rhaas 1505 GIC 119365 : fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1506 119365 : if (fd < 0)
3761 heikki.linnakangas 1507 ECB : {
3761 heikki.linnakangas 1508 UIC 0 : if (errno == ENOENT && missing_ok)
3761 heikki.linnakangas 1509 LBC 0 : return false;
4472 magnus 1510 UIC 0 : ereport(ERROR,
1511 : (errcode_for_file_access(),
1512 : errmsg("could not open file \"%s\": %m", readfilename)));
1513 : }
1514 :
520 rhaas 1515 GIC 119365 : _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1516 :
1826 magnus 1517 119364 : if (!noverify_checksums && DataChecksumsEnabled())
1832 magnus 1518 ECB : {
1519 : char *filename;
1520 :
1521 : /*
1522 : * Get the filename (excluding path). As last_dir_separator()
1523 : * includes the last directory separator, we chop that off by
1524 : * incrementing the pointer.
1525 : */
1832 magnus 1526 GIC 28387 : filename = last_dir_separator(readfilename) + 1;
1527 :
1832 magnus 1528 CBC 28387 : if (is_checksummed_file(readfilename, filename))
1832 magnus 1529 ECB : {
1832 magnus 1530 GIC 27858 : verify_checksum = true;
1832 magnus 1531 EUB :
1532 : /*
1533 : * Cut off at the segment boundary (".") to get the segment number
1534 : * in order to mix it into the checksum.
1535 : */
1832 magnus 1536 GIC 27858 : segmentpath = strstr(filename, ".");
1537 27858 : if (segmentpath != NULL)
1538 : {
1832 magnus 1539 UIC 0 : segmentno = atoi(segmentpath + 1);
1540 0 : if (segmentno == 0)
1541 0 : ereport(ERROR,
1542 : (errmsg("invalid segment number %d in file \"%s\"",
1543 : segmentno, filename)));
1544 : }
1545 : }
1832 magnus 1546 ECB : }
1547 :
1026 rhaas 1548 : /*
1549 : * Loop until we read the amount of data the caller told us to expect. The
1550 : * file could be longer, if it was extended while we were sending it, but
1551 : * for a base backup we can ignore such extended data. It will be restored
1552 : * from WAL.
1553 : */
1026 rhaas 1554 GIC 265468 : while (len < statbuf->st_size)
1555 : {
520 1556 146104 : size_t remaining = statbuf->st_size - len;
1557 :
1558 : /* Try to read some more data. */
1559 292208 : cnt = basebackup_read_file(fd, sink->bbs_buffer,
1560 146104 : Min(sink->bbs_buffer_length, remaining),
1026 rhaas 1561 ECB : len, readfilename, true);
1562 :
1832 magnus 1563 : /*
1564 : * The checksums are verified at block level, so we iterate over the
1809 tgl 1565 EUB : * buffer in chunks of BLCKSZ, after making sure that
1566 : * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1567 : * BLCKSZ bytes.
1568 : */
520 rhaas 1569 GIC 146104 : Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1832 magnus 1570 EUB :
1832 magnus 1571 GIC 146104 : if (verify_checksum && (cnt % BLCKSZ != 0))
1572 : {
1832 magnus 1573 LBC 0 : ereport(WARNING,
1574 : (errmsg("could not verify checksum in file \"%s\", block "
722 peter 1575 ECB : "%u: read buffer size %d and page size %d "
1576 : "differ",
1832 magnus 1577 : readfilename, blkno, (int) cnt, BLCKSZ)));
1832 magnus 1578 UIC 0 : verify_checksum = false;
1579 : }
1580 :
1832 magnus 1581 GIC 146104 : if (verify_checksum)
1582 : {
1583 118694 : for (i = 0; i < cnt / BLCKSZ; i++)
1584 : {
520 rhaas 1585 84496 : page = sink->bbs_buffer + BLCKSZ * i;
1586 :
1832 magnus 1587 ECB : /*
1588 : * Only check pages which have not been modified since the
1589 : * start of the base backup. Otherwise, they might have been
1590 : * written only halfway and the checksum would not be valid.
1591 : * However, replaying WAL would reinstate the correct page in
1592 : * this case. We also skip completely new pages, since they
1593 : * don't have a checksum yet.
1594 : */
520 rhaas 1595 GIC 84496 : if (!PageIsNew(page) && PageGetLSN(page) < sink->bbs_state->startptr)
1596 : {
1832 magnus 1597 84496 : checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1598 84496 : phdr = (PageHeader) page;
1599 84496 : if (phdr->pd_checksum != checksum)
1600 : {
1601 : /*
1602 : * Retry the block on the first failure. It's
1603 : * possible that we read the first 4K page of the
1604 : * block just before postgres updated the entire block
1605 : * so it ends up looking torn to us. If, before we
1606 : * retry the read, the concurrent write of the block
1607 : * finishes, the page LSN will be updated and we'll
1608 : * realize that we should ignore this block.
1609 : *
1610 : * There's no guarantee that this will actually
1611 : * happen, though: the torn write could take an
1612 : * arbitrarily long time to complete. Retrying multiple
1613 : * times wouldn't fix this problem, either, though
1614 : * it would reduce the chances of it happening in
1615 : * practice. The only real fix here seems to be to
1616 : * have some kind of interlock that allows us to wait
1617 : * until we can be certain that no write to the block
1618 : * is in progress. Since we don't have any such thing
1619 : * right now, we just do this and hope for the best.
1620 : */
1621 28 : if (block_retry == false)
1622 14 : {
1026 rhaas 1623 ECB : int reread_cnt;
1832 magnus 1624 :
1625 : /* Reread the failed block */
1626 : reread_cnt =
520 rhaas 1627 GIC 14 : basebackup_read_file(fd,
1628 14 : sink->bbs_buffer + BLCKSZ * i,
1026 rhaas 1629 CBC 14 : BLCKSZ, len + BLCKSZ * i,
1026 rhaas 1630 ECB : readfilename,
1631 : false);
1026 rhaas 1632 GIC 14 : if (reread_cnt == 0)
1633 : {
1311 rhaas 1634 ECB : /*
1635 : * If we hit end-of-file, a concurrent
1636 : * truncation must have occurred, so break out
1637 : * of this loop just as if the initial fread()
1638 : * returned 0. We'll drop through to the same
1639 : * code that handles that case. (We must fix
1640 : * up cnt first, though.)
1641 : */
1026 rhaas 1642 UIC 0 : cnt = BLCKSZ * i;
1643 0 : break;
1832 magnus 1644 EUB : }
1645 :
1646 : /* Set flag so we know a retry was attempted */
1832 magnus 1647 GIC 14 : block_retry = true;
1648 :
1832 magnus 1649 ECB : /* Reset loop to validate the block again */
1832 magnus 1650 GIC 14 : i--;
1651 14 : continue;
1832 magnus 1652 ECB : }
1653 :
1832 magnus 1654 GIC 14 : checksum_failures++;
1655 :
1832 magnus 1656 CBC 14 : if (checksum_failures <= 5)
1832 magnus 1657 GIC 12 : ereport(WARNING,
1832 magnus 1658 ECB : (errmsg("checksum verification failed in "
722 peter 1659 : "file \"%s\", block %u: calculated "
1660 : "%X but expected %X",
1661 : readfilename, blkno, checksum,
1662 : phdr->pd_checksum)));
1832 magnus 1663 GIC 14 : if (checksum_failures == 5)
1664 2 : ereport(WARNING,
1832 magnus 1665 ECB : (errmsg("further checksum verification "
1666 : "failures in file \"%s\" will not "
1667 : "be reported", readfilename)));
1668 : }
1669 : }
1832 magnus 1670 GIC 84482 : block_retry = false;
1671 84482 : blkno++;
1832 magnus 1672 ECB : }
1673 : }
1674 :
1675 : /*
1676 : * If we hit end-of-file, a concurrent truncation must have occurred.
1677 : * That's not an error condition, because WAL replay will fix things
1678 : * up.
1679 : */
66 rhaas 1680 GIC 146104 : if (cnt == 0)
66 rhaas 1681 UIC 0 : break;
66 rhaas 1682 ECB :
66 rhaas 1683 EUB : /* Archive the data we just read. */
520 rhaas 1684 GIC 146104 : bbsink_archive_contents(sink, cnt);
1685 :
1101 rhaas 1686 ECB : /* Also feed it to the checksum machinery. */
520 rhaas 1687 GIC 146104 : if (pg_checksum_update(&checksum_ctx,
1688 146104 : (uint8 *) sink->bbs_buffer, cnt) < 0)
858 michael 1689 LBC 0 : elog(ERROR, "could not update checksum of base backup");
1101 rhaas 1690 ECB :
4472 magnus 1691 GBC 146104 : len += cnt;
1692 : }
4472 magnus 1693 ECB :
1694 : /* If the file was truncated while we were sending it, pad it with zeros */
520 rhaas 1695 GIC 119364 : while (len < statbuf->st_size)
1696 : {
520 rhaas 1697 LBC 0 : size_t remaining = statbuf->st_size - len;
520 rhaas 1698 UIC 0 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
520 rhaas 1699 EUB :
520 rhaas 1700 UBC 0 : MemSet(sink->bbs_buffer, 0, nbytes);
520 rhaas 1701 UIC 0 : if (pg_checksum_update(&checksum_ctx,
520 rhaas 1702 UBC 0 : (uint8 *) sink->bbs_buffer,
520 rhaas 1703 EUB : nbytes) < 0)
520 rhaas 1704 UBC 0 : elog(ERROR, "could not update checksum of base backup");
520 rhaas 1705 UIC 0 : bbsink_archive_contents(sink, nbytes);
520 rhaas 1706 UBC 0 : len += nbytes;
4472 magnus 1707 EUB : }
1708 :
1709 : /*
1710 : * Pad to a block boundary, per tar format requirements. (This small piece
1711 : * of data is probably not worth throttling, and is not checksummed
1712 : * because it's not actually part of the file.)
1713 : */
520 rhaas 1714 GIC 119364 : _tarWritePadding(sink, len);
1715 :
1026 rhaas 1716 CBC 119364 : CloseTransientFile(fd);
1717 :
1832 magnus 1718 119364 : if (checksum_failures > 1)
1719 : {
1720 2 : ereport(WARNING,
1721 : (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1294 peter 1722 ECB : "file \"%s\" has a total of %d checksum verification failures",
1723 : checksum_failures,
1724 : readfilename, checksum_failures)));
1725 :
1458 magnus 1726 GIC 2 : pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1727 : }
1458 magnus 1728 ECB :
1832 magnus 1729 GIC 119364 : total_checksum_failures += checksum_failures;
1730 :
1081 rhaas 1731 CBC 119364 : AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1081 rhaas 1732 GIC 119364 : (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1101 rhaas 1733 ECB :
3761 heikki.linnakangas 1734 CBC 119364 : return true;
1735 : }
4472 magnus 1736 ECB :
1737 : static int64
520 rhaas 1738 GIC 125914 : _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
1739 : struct stat *statbuf, bool sizeonly)
4472 magnus 1740 ECB : {
1741 : enum tarError rc;
1742 :
2384 peter_e 1743 GIC 125914 : if (!sizeonly)
1744 : {
520 rhaas 1745 ECB : /*
1746 : * As of this writing, the smallest supported block size is 1kB, which
1747 : * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
1748 : * multiple of BLCKSZ, it should be safe to assume that the buffer is
1749 : * large enough to fit an entire tar block. We double-check by means
1750 : * of these assertions.
1751 : */
1752 : StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
1753 : "BLCKSZ too small for tar block");
520 rhaas 1754 GIC 122684 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
1755 :
520 rhaas 1756 CBC 122684 : rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
1757 : statbuf->st_size, statbuf->st_mode,
520 rhaas 1758 ECB : statbuf->st_uid, statbuf->st_gid,
1759 : statbuf->st_mtime);
1760 :
2384 peter_e 1761 GIC 122684 : switch (rc)
1762 : {
2384 peter_e 1763 CBC 122683 : case TAR_OK:
2384 peter_e 1764 GIC 122683 : break;
2384 peter_e 1765 CBC 1 : case TAR_NAME_TOO_LONG:
1766 1 : ereport(ERROR,
2384 peter_e 1767 ECB : (errmsg("file name too long for tar format: \"%s\"",
1768 : filename)));
1769 : break;
2384 peter_e 1770 UIC 0 : case TAR_SYMLINK_TOO_LONG:
1771 0 : ereport(ERROR,
2118 tgl 1772 EUB : (errmsg("symbolic link target too long for tar format: "
1773 : "file name \"%s\", target \"%s\"",
1774 : filename, linktarget)));
1775 : break;
2384 peter_e 1776 UIC 0 : default:
1777 0 : elog(ERROR, "unrecognized tar error: %d", rc);
2384 peter_e 1778 EUB : }
1779 :
520 rhaas 1780 GIC 122683 : bbsink_archive_contents(sink, TAR_BLOCK_SIZE);
1781 : }
2966 peter_e 1782 ECB :
520 rhaas 1783 GIC 125913 : return TAR_BLOCK_SIZE;
1784 : }
2384 peter_e 1785 ECB :
1786 : /*
1787 : * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
1788 : */
1789 : static void
520 rhaas 1790 GIC 119534 : _tarWritePadding(bbsink *sink, int len)
1791 : {
520 rhaas 1792 CBC 119534 : int pad = tarPaddingBytesRequired(len);
1793 :
2042 alvherre 1794 ECB : /*
1795 : * As in _tarWriteHeader, it should be safe to assume that the buffer is
1796 : * large enough that we don't need to do this in multiple chunks.
1797 : */
520 rhaas 1798 GIC 119534 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
1799 119534 : Assert(pad <= TAR_BLOCK_SIZE);
3328 alvherre 1800 ECB :
520 rhaas 1801 CBC 119534 : if (pad > 0)
1802 : {
1803 22769 : MemSet(sink->bbs_buffer, 0, pad);
520 rhaas 1804 GIC 1768 : bbsink_archive_contents(sink, pad);
3328 alvherre 1805 ECB : }
3328 alvherre 1806 CBC 119534 : }
1807 :
1132 fujii 1808 ECB : /*
1809 : * If the entry in statbuf is a link, then adjust statbuf to make it look like a
1810 : * directory, so that it will be written that way.
1811 : */
1812 : static void
520 rhaas 1813 GIC 1990 : convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
1814 : {
520 rhaas 1815 ECB : /* If symlink, write it as a directory anyway */
520 rhaas 1816 GIC 1990 : if (S_ISLNK(statbuf->st_mode))
1817 66 : statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1132 fujii 1818 1990 : }
1819 :
1820 : /*
1821 : * Read some data from a file, setting a wait event and reporting any error
1822 : * encountered.
1823 : *
1824 : * If partial_read_ok is false, also report an error if the number of bytes
1825 : * read is not equal to the number of bytes requested.
1826 : *
1827 : * Returns the number of bytes read.
1026 rhaas 1828 ECB : */
1829 : static int
1026 rhaas 1830 GIC 156358 : basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
1831 : const char *filename, bool partial_read_ok)
1832 : {
1026 rhaas 1833 ECB : int rc;
1834 :
1026 rhaas 1835 CBC 156358 : pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
192 tmunro 1836 GIC 156358 : rc = pg_pread(fd, buf, nbytes, offset);
1026 rhaas 1837 CBC 156358 : pgstat_report_wait_end();
1026 rhaas 1838 EUB :
1026 rhaas 1839 GIC 156358 : if (rc < 0)
1026 rhaas 1840 UIC 0 : ereport(ERROR,
1026 rhaas 1841 ECB : (errcode_for_file_access(),
1026 rhaas 1842 EUB : errmsg("could not read file \"%s\": %m", filename)));
1026 rhaas 1843 GIC 156358 : if (!partial_read_ok && rc > 0 && rc != nbytes)
1026 rhaas 1844 UIC 0 : ereport(ERROR,
1845 : (errcode_for_file_access(),
1846 : errmsg("could not read file \"%s\": read %d of %zu",
1026 rhaas 1847 ECB : filename, rc, nbytes)));
1848 :
1026 rhaas 1849 GIC 156358 : return rc;
1850 : }
|
