Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * pg_combinebackup.c
4 : : * Combine incremental backups with prior backups.
5 : : *
6 : : * Copyright (c) 2017-2024, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/bin/pg_combinebackup/pg_combinebackup.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres_fe.h"
14 : :
15 : : #include <dirent.h>
16 : : #include <fcntl.h>
17 : : #include <limits.h>
18 : :
19 : : #include "backup_label.h"
20 : : #include "common/blkreftable.h"
21 : : #include "common/checksum_helper.h"
22 : : #include "common/controldata_utils.h"
23 : : #include "common/file_perm.h"
24 : : #include "common/file_utils.h"
25 : : #include "common/logging.h"
26 : : #include "copy_file.h"
27 : : #include "fe_utils/option_utils.h"
28 : : #include "getopt_long.h"
29 : : #include "lib/stringinfo.h"
30 : : #include "load_manifest.h"
31 : : #include "reconstruct.h"
32 : : #include "write_manifest.h"
33 : :
34 : : /* Incremental file naming convention. */
35 : : #define INCREMENTAL_PREFIX "INCREMENTAL."
36 : : #define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
37 : :
38 : : /*
39 : : * Tracking for directories that need to be removed, or have their contents
40 : : * removed, if the operation fails.
41 : : */
42 : : typedef struct cb_cleanup_dir
43 : : {
44 : : char *target_path;
45 : : bool rmtopdir;
46 : : struct cb_cleanup_dir *next;
47 : : } cb_cleanup_dir;
48 : :
49 : : /*
50 : : * Stores a tablespace mapping provided using -T, --tablespace-mapping.
51 : : */
52 : : typedef struct cb_tablespace_mapping
53 : : {
54 : : char old_dir[MAXPGPATH];
55 : : char new_dir[MAXPGPATH];
56 : : struct cb_tablespace_mapping *next;
57 : : } cb_tablespace_mapping;
58 : :
59 : : /*
60 : : * Stores data parsed from all command-line options.
61 : : */
62 : : typedef struct cb_options
63 : : {
64 : : bool debug;
65 : : char *output;
66 : : bool dry_run;
67 : : bool no_sync;
68 : : cb_tablespace_mapping *tsmappings;
69 : : pg_checksum_type manifest_checksums;
70 : : bool no_manifest;
71 : : DataDirSyncMethod sync_method;
72 : : CopyMethod copy_method;
73 : : } cb_options;
74 : :
75 : : /*
76 : : * Data about a tablespace.
77 : : *
78 : : * Every normal tablespace needs a tablespace mapping, but in-place tablespaces
79 : : * don't, so the list of tablespaces can contain more entries than the list of
80 : : * tablespace mappings.
81 : : */
82 : : typedef struct cb_tablespace
83 : : {
84 : : Oid oid;
85 : : bool in_place;
86 : : char old_dir[MAXPGPATH];
87 : : char new_dir[MAXPGPATH];
88 : : struct cb_tablespace *next;
89 : : } cb_tablespace;
90 : :
91 : : /* Directories to be removed if we exit uncleanly. */
92 : : cb_cleanup_dir *cleanup_dir_list = NULL;
93 : :
94 : : static void add_tablespace_mapping(cb_options *opt, char *arg);
95 : : static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
96 : : static uint64 check_control_files(int n_backups, char **backup_dirs);
97 : : static void check_input_dir_permissions(char *dir);
98 : : static void cleanup_directories_atexit(void);
99 : : static void create_output_directory(char *dirname, cb_options *opt);
100 : : static void help(const char *progname);
101 : : static bool parse_oid(char *s, Oid *result);
102 : : static void process_directory_recursively(Oid tsoid,
103 : : char *input_directory,
104 : : char *output_directory,
105 : : char *relative_path,
106 : : int n_prior_backups,
107 : : char **prior_backup_dirs,
108 : : manifest_data **manifests,
109 : : manifest_writer *mwriter,
110 : : cb_options *opt);
111 : : static int read_pg_version_file(char *directory);
112 : : static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
113 : : static void reset_directory_cleanup_list(void);
114 : : static cb_tablespace *scan_for_existing_tablespaces(char *pathname,
115 : : cb_options *opt);
116 : : static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen);
117 : :
118 : : /*
119 : : * Main program.
120 : : */
121 : : int
116 rhaas@postgresql.org 122 :GNC 22 : main(int argc, char *argv[])
123 : : {
124 : : static struct option long_options[] = {
125 : : {"debug", no_argument, NULL, 'd'},
126 : : {"dry-run", no_argument, NULL, 'n'},
127 : : {"no-sync", no_argument, NULL, 'N'},
128 : : {"output", required_argument, NULL, 'o'},
129 : : {"tablespace-mapping", no_argument, NULL, 'T'},
130 : : {"manifest-checksums", required_argument, NULL, 1},
131 : : {"no-manifest", no_argument, NULL, 2},
132 : : {"sync-method", required_argument, NULL, 3},
133 : : {"clone", no_argument, NULL, 4},
134 : : {"copy-file-range", no_argument, NULL, 5},
135 : : {NULL, 0, NULL, 0}
136 : : };
137 : :
138 : : const char *progname;
139 : : char *last_input_dir;
140 : : int i;
141 : : int optindex;
142 : : int c;
143 : : int n_backups;
144 : : int n_prior_backups;
145 : : int version;
146 : : uint64 system_identifier;
147 : : char **prior_backup_dirs;
148 : : cb_options opt;
149 : : cb_tablespace *tablespaces;
150 : : cb_tablespace *ts;
151 : : StringInfo last_backup_label;
152 : : manifest_data **manifests;
153 : : manifest_writer *mwriter;
154 : :
155 : 22 : pg_logging_init(argv[0]);
156 : 22 : progname = get_progname(argv[0]);
5 michael@paquier.xyz 157 : 22 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
116 rhaas@postgresql.org 158 : 22 : handle_help_version_opts(argc, argv, progname, help);
159 : :
160 : 20 : memset(&opt, 0, sizeof(opt));
161 : 20 : opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
162 : 20 : opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
9 tomas.vondra@postgre 163 : 20 : opt.copy_method = COPY_METHOD_COPY;
164 : :
165 : : /* process command-line options */
116 rhaas@postgresql.org 166 : 47 : while ((c = getopt_long(argc, argv, "dnNPo:T:",
167 [ + + ]: 47 : long_options, &optindex)) != -1)
168 : : {
169 [ + - + + : 28 : switch (c)
- + + - -
- + ]
170 : : {
171 : 4 : case 'd':
172 : 4 : opt.debug = true;
173 : 4 : pg_logging_increase_verbosity();
174 : 4 : break;
116 rhaas@postgresql.org 175 :UNC 0 : case 'n':
176 : 0 : opt.dry_run = true;
177 : 0 : break;
116 rhaas@postgresql.org 178 :GNC 3 : case 'N':
179 : 3 : opt.no_sync = true;
180 : 3 : break;
181 : 17 : case 'o':
182 : 17 : opt.output = optarg;
183 : 17 : break;
116 rhaas@postgresql.org 184 :UNC 0 : case 'T':
185 : 0 : add_tablespace_mapping(&opt, optarg);
186 : 0 : break;
116 rhaas@postgresql.org 187 :GNC 2 : case 1:
188 [ - + ]: 2 : if (!pg_checksum_parse_type(optarg,
189 : : &opt.manifest_checksums))
116 rhaas@postgresql.org 190 :UNC 0 : pg_fatal("unrecognized checksum algorithm: \"%s\"",
191 : : optarg);
116 rhaas@postgresql.org 192 :GNC 2 : break;
193 : 1 : case 2:
194 : 1 : opt.no_manifest = true;
195 : 1 : break;
116 rhaas@postgresql.org 196 :UNC 0 : case 3:
197 [ # # ]: 0 : if (!parse_sync_method(optarg, &opt.sync_method))
198 : 0 : exit(1);
199 : 0 : break;
9 tomas.vondra@postgre 200 : 0 : case 4:
201 : 0 : opt.copy_method = COPY_METHOD_CLONE;
202 : 0 : break;
203 : 0 : case 5:
204 : 0 : opt.copy_method = COPY_METHOD_COPY_FILE_RANGE;
205 : 0 : break;
116 rhaas@postgresql.org 206 :GNC 1 : default:
207 : : /* getopt_long already emitted a complaint */
208 : 1 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
209 : 1 : exit(1);
210 : : }
211 : : }
212 : :
213 [ + + ]: 19 : if (optind >= argc)
214 : : {
215 : 1 : pg_log_error("%s: no input directories specified", progname);
216 : 1 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
217 : 1 : exit(1);
218 : : }
219 : :
220 [ + + ]: 18 : if (opt.output == NULL)
221 : 1 : pg_fatal("no output directory specified");
222 : :
223 : : /* If no manifest is needed, no checksums are needed, either. */
224 [ + + ]: 17 : if (opt.no_manifest)
225 : 1 : opt.manifest_checksums = CHECKSUM_TYPE_NONE;
226 : :
227 : : /* Check that the platform supports the requested copy method. */
9 tomas.vondra@postgre 228 [ - + ]: 17 : if (opt.copy_method == COPY_METHOD_CLONE)
229 : : {
230 : : #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
231 : : (defined(__linux__) && defined(FICLONE))
232 : :
233 : : if (opt.dry_run)
234 : : pg_log_debug("would use cloning to copy files");
235 : : else
236 : : pg_log_debug("will use cloning to copy files");
237 : :
238 : : #else
9 tomas.vondra@postgre 239 :UNC 0 : pg_fatal("file cloning not supported on this platform");
240 : : #endif
241 : : }
9 tomas.vondra@postgre 242 [ - + ]:GNC 17 : else if (opt.copy_method == COPY_METHOD_COPY_FILE_RANGE)
243 : : {
244 : : #if defined(HAVE_COPY_FILE_RANGE)
245 : :
9 tomas.vondra@postgre 246 [ # # ]:UNC 0 : if (opt.dry_run)
247 [ # # ]: 0 : pg_log_debug("would use copy_file_range to copy blocks");
248 : : else
249 [ # # ]: 0 : pg_log_debug("will use copy_file_range to copy blocks");
250 : :
251 : : #else
252 : : pg_fatal("copy_file_range not supported on this platform");
253 : : #endif
254 : : }
255 : :
256 : : /* Read the server version from the final backup. */
116 rhaas@postgresql.org 257 :GNC 17 : version = read_pg_version_file(argv[argc - 1]);
258 : :
259 : : /* Sanity-check control files. */
260 : 17 : n_backups = argc - optind;
32 261 : 17 : system_identifier = check_control_files(n_backups, argv + optind);
262 : :
263 : : /* Sanity-check backup_label files, and get the contents of the last one. */
116 264 : 16 : last_backup_label = check_backup_label_files(n_backups, argv + optind);
265 : :
266 : : /*
267 : : * We'll need the pathnames to the prior backups. By "prior" we mean all
268 : : * but the last one listed on the command line.
269 : : */
270 : 11 : n_prior_backups = argc - optind - 1;
271 : 11 : prior_backup_dirs = argv + optind;
272 : :
273 : : /* Load backup manifests. */
274 : 11 : manifests = load_backup_manifests(n_backups, prior_backup_dirs);
275 : :
276 : : /*
277 : : * Validate the manifest system identifier against the backup system
278 : : * identifier.
279 : : */
32 280 [ + + ]: 31 : for (i = 0; i < n_backups; i++)
281 : : {
282 [ + - ]: 21 : if (manifests[i] &&
283 [ + + ]: 21 : manifests[i]->system_identifier != system_identifier)
284 : : {
285 : : char *controlpath;
286 : :
287 : 1 : controlpath = psprintf("%s/%s", prior_backup_dirs[i], "global/pg_control");
288 : :
289 : 1 : pg_fatal("%s: manifest system identifier is %llu, but control file has %llu",
290 : : controlpath,
291 : : (unsigned long long) manifests[i]->system_identifier,
292 : : (unsigned long long) system_identifier);
293 : : }
294 : : }
295 : :
296 : : /* Figure out which tablespaces are going to be included in the output. */
116 297 : 10 : last_input_dir = argv[argc - 1];
298 : 10 : check_input_dir_permissions(last_input_dir);
299 : 10 : tablespaces = scan_for_existing_tablespaces(last_input_dir, &opt);
300 : :
301 : : /*
302 : : * Create output directories.
303 : : *
304 : : * We create one output directory for the main data directory plus one for
305 : : * each non-in-place tablespace. create_output_directory() will arrange
306 : : * for those directories to be cleaned up on failure. In-place tablespaces
307 : : * aren't handled at this stage because they're located beneath the main
308 : : * output directory, and thus the cleanup of that directory will get rid
309 : : * of them. Plus, the pg_tblspc directory that needs to contain them
310 : : * doesn't exist yet.
311 : : */
312 : 10 : atexit(cleanup_directories_atexit);
313 : 10 : create_output_directory(opt.output, &opt);
314 [ - + ]: 10 : for (ts = tablespaces; ts != NULL; ts = ts->next)
116 rhaas@postgresql.org 315 [ # # ]:UNC 0 : if (!ts->in_place)
316 : 0 : create_output_directory(ts->new_dir, &opt);
317 : :
318 : : /* If we need to write a backup_manifest, prepare to do so. */
116 rhaas@postgresql.org 319 [ + - + + ]:GNC 10 : if (!opt.dry_run && !opt.no_manifest)
320 : : {
32 321 : 9 : mwriter = create_manifest_writer(opt.output, system_identifier);
322 : :
323 : : /*
324 : : * Verify that we have a backup manifest for the final backup; else we
325 : : * won't have the WAL ranges for the resulting manifest.
326 : : */
116 327 [ - + ]: 9 : if (manifests[n_prior_backups] == NULL)
116 rhaas@postgresql.org 328 :UNC 0 : pg_fatal("can't generate a manifest because no manifest is available for the final input backup");
329 : : }
330 : : else
116 rhaas@postgresql.org 331 :GNC 1 : mwriter = NULL;
332 : :
333 : : /* Write backup label into output directory. */
334 [ - + ]: 10 : if (opt.dry_run)
116 rhaas@postgresql.org 335 [ # # ]:UNC 0 : pg_log_debug("would generate \"%s/backup_label\"", opt.output);
336 : : else
337 : : {
116 rhaas@postgresql.org 338 [ + + ]:GNC 10 : pg_log_debug("generating \"%s/backup_label\"", opt.output);
339 : 10 : last_backup_label->cursor = 0;
340 : 10 : write_backup_label(opt.output, last_backup_label,
341 : : opt.manifest_checksums, mwriter);
342 : : }
343 : :
344 : : /* Process everything that's not part of a user-defined tablespace. */
345 [ + + ]: 10 : pg_log_debug("processing backup directory \"%s\"", last_input_dir);
346 : 10 : process_directory_recursively(InvalidOid, last_input_dir, opt.output,
347 : : NULL, n_prior_backups, prior_backup_dirs,
348 : : manifests, mwriter, &opt);
349 : :
350 : : /* Process user-defined tablespaces. */
351 [ - + ]: 10 : for (ts = tablespaces; ts != NULL; ts = ts->next)
352 : : {
116 rhaas@postgresql.org 353 [ # # ]:UNC 0 : pg_log_debug("processing tablespace directory \"%s\"", ts->old_dir);
354 : :
355 : : /*
356 : : * If it's a normal tablespace, we need to set up a symbolic link from
357 : : * pg_tblspc/${OID} to the target directory; if it's an in-place
358 : : * tablespace, we need to create a directory at pg_tblspc/${OID}.
359 : : */
360 [ # # ]: 0 : if (!ts->in_place)
361 : : {
362 : : char linkpath[MAXPGPATH];
363 : :
364 : 0 : snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output,
365 : : ts->oid);
366 : :
367 [ # # ]: 0 : if (opt.dry_run)
368 [ # # ]: 0 : pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
369 : : linkpath, ts->new_dir);
370 : : else
371 : : {
372 [ # # ]: 0 : pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
373 : : linkpath, ts->new_dir);
374 [ # # ]: 0 : if (symlink(ts->new_dir, linkpath) != 0)
375 : 0 : pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
376 : : linkpath, ts->new_dir);
377 : : }
378 : : }
379 : : else
380 : : {
381 [ # # ]: 0 : if (opt.dry_run)
382 [ # # ]: 0 : pg_log_debug("would create directory \"%s\"", ts->new_dir);
383 : : else
384 : : {
385 [ # # ]: 0 : pg_log_debug("creating directory \"%s\"", ts->new_dir);
386 [ # # ]: 0 : if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
387 : 0 : pg_fatal("could not create directory \"%s\": %m",
388 : : ts->new_dir);
389 : : }
390 : : }
391 : :
392 : : /* OK, now handle the directory contents. */
393 : 0 : process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
394 : : NULL, n_prior_backups, prior_backup_dirs,
395 : : manifests, mwriter, &opt);
396 : : }
397 : :
398 : : /* Finalize the backup_manifest, if we're generating one. */
116 rhaas@postgresql.org 399 [ + + ]:GNC 10 : if (mwriter != NULL)
400 : 9 : finalize_manifest(mwriter,
401 : 9 : manifests[n_prior_backups]->first_wal_range);
402 : :
403 : : /* fsync that output directory unless we've been told not to do so */
404 [ + + ]: 10 : if (!opt.no_sync)
405 : : {
406 [ - + ]: 7 : if (opt.dry_run)
116 rhaas@postgresql.org 407 [ # # ]:UNC 0 : pg_log_debug("would recursively fsync \"%s\"", opt.output);
408 : : else
409 : : {
116 rhaas@postgresql.org 410 [ + + ]:GNC 7 : pg_log_debug("recursively fsyncing \"%s\"", opt.output);
411 : 7 : sync_pgdata(opt.output, version * 10000, opt.sync_method);
412 : : }
413 : : }
414 : :
415 : : /* It's a success, so don't remove the output directories. */
416 : 10 : reset_directory_cleanup_list();
417 : 10 : exit(0);
418 : : }
419 : :
420 : : /*
421 : : * Process the option argument for the -T, --tablespace-mapping switch.
422 : : */
423 : : static void
116 rhaas@postgresql.org 424 :UNC 0 : add_tablespace_mapping(cb_options *opt, char *arg)
425 : : {
426 : 0 : cb_tablespace_mapping *tsmap = pg_malloc0(sizeof(cb_tablespace_mapping));
427 : : char *dst;
428 : : char *dst_ptr;
429 : : char *arg_ptr;
430 : :
431 : : /*
432 : : * Basically, we just want to copy everything before the equals sign to
433 : : * tsmap->old_dir and everything afterwards to tsmap->new_dir, but if
434 : : * there's more or less than one equals sign, that's an error, and if
435 : : * there's an equals sign preceded by a backslash, don't treat it as a
436 : : * field separator but instead copy a literal equals sign.
437 : : */
438 : 0 : dst_ptr = dst = tsmap->old_dir;
439 [ # # ]: 0 : for (arg_ptr = arg; *arg_ptr != '\0'; arg_ptr++)
440 : : {
441 [ # # ]: 0 : if (dst_ptr - dst >= MAXPGPATH)
442 : 0 : pg_fatal("directory name too long");
443 : :
444 [ # # # # ]: 0 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
445 : : ; /* skip backslash escaping = */
446 [ # # # # : 0 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
# # ]
447 : : {
448 [ # # ]: 0 : if (tsmap->new_dir[0] != '\0')
449 : 0 : pg_fatal("multiple \"=\" signs in tablespace mapping");
450 : : else
451 : 0 : dst = dst_ptr = tsmap->new_dir;
452 : : }
453 : : else
454 : 0 : *dst_ptr++ = *arg_ptr;
455 : : }
456 [ # # # # ]: 0 : if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
457 : 0 : pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
458 : :
459 : : /*
460 : : * All tablespaces are created with absolute directories, so specifying a
461 : : * non-absolute path here would never match, possibly confusing users.
462 : : *
463 : : * In contrast to pg_basebackup, both the old and new directories are on
464 : : * the local machine, so the local machine's definition of an absolute
465 : : * path is the only relevant one.
466 : : */
467 [ # # ]: 0 : if (!is_absolute_path(tsmap->old_dir))
468 : 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
469 : : tsmap->old_dir);
470 : :
471 [ # # ]: 0 : if (!is_absolute_path(tsmap->new_dir))
472 : 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
473 : : tsmap->new_dir);
474 : :
475 : : /* Canonicalize paths to avoid spurious failures when comparing. */
476 : 0 : canonicalize_path(tsmap->old_dir);
477 : 0 : canonicalize_path(tsmap->new_dir);
478 : :
479 : : /* Add it to the list. */
480 : 0 : tsmap->next = opt->tsmappings;
481 : 0 : opt->tsmappings = tsmap;
482 : 0 : }
483 : :
484 : : /*
485 : : * Check that the backup_label files form a coherent backup chain, and return
486 : : * the contents of the backup_label file from the latest backup.
487 : : */
488 : : static StringInfo
116 rhaas@postgresql.org 489 :GNC 16 : check_backup_label_files(int n_backups, char **backup_dirs)
490 : : {
491 : 16 : StringInfo buf = makeStringInfo();
492 : 16 : StringInfo lastbuf = buf;
493 : : int i;
494 : 16 : TimeLineID check_tli = 0;
495 : 16 : XLogRecPtr check_lsn = InvalidXLogRecPtr;
496 : :
497 : : /* Try to read each backup_label file in turn, last to first. */
498 [ + + ]: 42 : for (i = n_backups - 1; i >= 0; --i)
499 : : {
500 : : char pathbuf[MAXPGPATH];
501 : : int fd;
502 : : TimeLineID start_tli;
503 : : TimeLineID previous_tli;
504 : : XLogRecPtr start_lsn;
505 : : XLogRecPtr previous_lsn;
506 : :
507 : : /* Open the backup_label file. */
508 : 31 : snprintf(pathbuf, MAXPGPATH, "%s/backup_label", backup_dirs[i]);
509 [ + + ]: 31 : pg_log_debug("reading \"%s\"", pathbuf);
510 [ - + ]: 31 : if ((fd = open(pathbuf, O_RDONLY, 0)) < 0)
116 rhaas@postgresql.org 511 :UNC 0 : pg_fatal("could not open file \"%s\": %m", pathbuf);
512 : :
513 : : /*
514 : : * Slurp the whole file into memory.
515 : : *
516 : : * The exact size limit that we impose here doesn't really matter --
517 : : * most of what's supposed to be in the file is fixed size and quite
518 : : * short. However, the length of the backup_label is limited (at least
519 : : * by some parts of the code) to MAXPGPATH, so include that value in
520 : : * the maximum length that we tolerate.
521 : : */
116 rhaas@postgresql.org 522 :GNC 31 : slurp_file(fd, pathbuf, buf, 10000 + MAXPGPATH);
523 : :
524 : : /* Close the file. */
525 [ - + ]: 31 : if (close(fd) != 0)
116 rhaas@postgresql.org 526 :UNC 0 : pg_fatal("could not close \"%s\": %m", pathbuf);
527 : :
528 : : /* Parse the file contents. */
116 rhaas@postgresql.org 529 :GNC 31 : parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
530 : : &previous_tli, &previous_lsn);
531 : :
532 : : /*
533 : : * Sanity checks.
534 : : *
535 : : * XXX. It's actually not required that start_lsn == check_lsn. It
536 : : * would be OK if start_lsn > check_lsn provided that start_lsn is
537 : : * less than or equal to the relevant switchpoint. But at the moment
538 : : * we don't have that information.
539 : : */
540 [ + + + + ]: 31 : if (i > 0 && previous_tli == 0)
541 : 1 : pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
542 : : backup_dirs[i]);
543 [ + + + + ]: 30 : if (i == 0 && previous_tli != 0)
544 : 1 : pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
545 : : backup_dirs[i]);
546 [ + + - + ]: 29 : if (i < n_backups - 1 && start_tli != check_tli)
116 rhaas@postgresql.org 547 :UNC 0 : pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
548 : : backup_dirs[i], start_tli, check_tli);
116 rhaas@postgresql.org 549 [ + + + + ]:GNC 29 : if (i < n_backups - 1 && start_lsn != check_lsn)
550 : 3 : pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
551 : : backup_dirs[i],
552 : : LSN_FORMAT_ARGS(start_lsn),
553 : : LSN_FORMAT_ARGS(check_lsn));
554 : 26 : check_tli = previous_tli;
555 : 26 : check_lsn = previous_lsn;
556 : :
557 : : /*
558 : : * The last backup label in the chain needs to be saved for later use,
559 : : * while the others are only needed within this loop.
560 : : */
561 [ + + ]: 26 : if (lastbuf == buf)
562 : 15 : buf = makeStringInfo();
563 : : else
564 : 11 : resetStringInfo(buf);
565 : : }
566 : :
567 : : /* Free memory that we don't need any more. */
568 [ + - ]: 11 : if (lastbuf != buf)
29 dgustafsson@postgres 569 : 11 : destroyStringInfo(buf);
570 : :
571 : : /*
572 : : * Return the data from the first backup_info that we read (which is the
573 : : * backup_label from the last directory specified on the command line).
574 : : */
116 rhaas@postgresql.org 575 : 11 : return lastbuf;
576 : : }
577 : :
578 : : /*
579 : : * Sanity check control files and return system_identifier.
580 : : */
581 : : static uint64
582 : 17 : check_control_files(int n_backups, char **backup_dirs)
583 : : {
584 : : int i;
585 : 17 : uint64 system_identifier = 0; /* placate compiler */
586 : :
587 : : /* Try to read each control file in turn, last to first. */
588 [ + + ]: 51 : for (i = n_backups - 1; i >= 0; --i)
589 : : {
590 : : ControlFileData *control_file;
591 : : bool crc_ok;
592 : : char *controlpath;
593 : :
594 : 35 : controlpath = psprintf("%s/%s", backup_dirs[i], "global/pg_control");
595 [ + + ]: 35 : pg_log_debug("reading \"%s\"", controlpath);
32 596 : 35 : control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
597 : :
598 : : /* Control file contents not meaningful if CRC is bad. */
116 599 [ - + ]: 35 : if (!crc_ok)
109 john.naylor@postgres 600 :UNC 0 : pg_fatal("%s: CRC is incorrect", controlpath);
601 : :
602 : : /* Can't interpret control file if not current version. */
116 rhaas@postgresql.org 603 [ - + ]:GNC 35 : if (control_file->pg_control_version != PG_CONTROL_VERSION)
116 rhaas@postgresql.org 604 :UNC 0 : pg_fatal("%s: unexpected control file version",
605 : : controlpath);
606 : :
607 : : /* System identifiers should all match. */
116 rhaas@postgresql.org 608 [ + + ]:GNC 35 : if (i == n_backups - 1)
609 : 17 : system_identifier = control_file->system_identifier;
610 [ + + ]: 18 : else if (system_identifier != control_file->system_identifier)
611 : 1 : pg_fatal("%s: expected system identifier %llu, but found %llu",
612 : : controlpath, (unsigned long long) system_identifier,
613 : : (unsigned long long) control_file->system_identifier);
614 : :
615 : : /* Release memory. */
616 : 34 : pfree(control_file);
617 : 34 : pfree(controlpath);
618 : : }
619 : :
620 : : /*
621 : : * If debug output is enabled, make a note of the system identifier that
622 : : * we found in all of the relevant control files.
623 : : */
624 [ + + ]: 16 : pg_log_debug("system identifier is %llu",
625 : : (unsigned long long) system_identifier);
626 : :
32 627 : 16 : return system_identifier;
628 : : }
629 : :
630 : : /*
631 : : * Set default permissions for new files and directories based on the
632 : : * permissions of the given directory. The intent here is that the output
633 : : * directory should use the same permissions scheme as the final input
634 : : * directory.
635 : : */
636 : : static void
116 637 : 10 : check_input_dir_permissions(char *dir)
638 : : {
639 : : struct stat st;
640 : :
641 [ - + ]: 10 : if (stat(dir, &st) != 0)
116 rhaas@postgresql.org 642 :UNC 0 : pg_fatal("could not stat \"%s\": %m", dir);
643 : :
116 rhaas@postgresql.org 644 :GNC 10 : SetDataDirectoryCreatePerm(st.st_mode);
645 : 10 : }
646 : :
647 : : /*
648 : : * Clean up output directories before exiting.
649 : : */
650 : : static void
651 : 10 : cleanup_directories_atexit(void)
652 : : {
653 [ - + ]: 10 : while (cleanup_dir_list != NULL)
654 : : {
116 rhaas@postgresql.org 655 :UNC 0 : cb_cleanup_dir *dir = cleanup_dir_list;
656 : :
657 [ # # ]: 0 : if (dir->rmtopdir)
658 : : {
659 : 0 : pg_log_info("removing output directory \"%s\"", dir->target_path);
660 [ # # ]: 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
661 : 0 : pg_log_error("failed to remove output directory");
662 : : }
663 : : else
664 : : {
665 : 0 : pg_log_info("removing contents of output directory \"%s\"",
666 : : dir->target_path);
667 [ # # ]: 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
668 : 0 : pg_log_error("failed to remove contents of output directory");
669 : : }
670 : :
671 : 0 : cleanup_dir_list = cleanup_dir_list->next;
672 : 0 : pfree(dir);
673 : : }
116 rhaas@postgresql.org 674 :GNC 10 : }
675 : :
676 : : /*
677 : : * Create the named output directory, unless it already exists or we're in
678 : : * dry-run mode. If it already exists but is not empty, that's a fatal error.
679 : : *
680 : : * Adds the created directory to the list of directories to be cleaned up
681 : : * at process exit.
682 : : */
683 : : static void
684 : 10 : create_output_directory(char *dirname, cb_options *opt)
685 : : {
686 [ + - - - : 10 : switch (pg_check_dir(dirname))
- ]
687 : : {
688 : 10 : case 0:
689 [ - + ]: 10 : if (opt->dry_run)
690 : : {
116 rhaas@postgresql.org 691 [ # # ]:UNC 0 : pg_log_debug("would create directory \"%s\"", dirname);
692 : 0 : return;
693 : : }
116 rhaas@postgresql.org 694 [ + + ]:GNC 10 : pg_log_debug("creating directory \"%s\"", dirname);
695 [ - + ]: 10 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
116 rhaas@postgresql.org 696 :UNC 0 : pg_fatal("could not create directory \"%s\": %m", dirname);
116 rhaas@postgresql.org 697 :GNC 10 : remember_to_cleanup_directory(dirname, true);
698 : 10 : break;
699 : :
116 rhaas@postgresql.org 700 :UNC 0 : case 1:
701 [ # # ]: 0 : pg_log_debug("using existing directory \"%s\"", dirname);
702 : 0 : remember_to_cleanup_directory(dirname, false);
703 : 0 : break;
704 : :
705 : 0 : case 2:
706 : : case 3:
707 : : case 4:
708 : 0 : pg_fatal("directory \"%s\" exists but is not empty", dirname);
709 : :
710 : 0 : case -1:
711 : 0 : pg_fatal("could not access directory \"%s\": %m", dirname);
712 : : }
713 : : }
714 : :
715 : : /*
716 : : * help
717 : : *
718 : : * Prints help page for the program
719 : : *
720 : : * progname: the name of the executed program, such as "pg_combinebackup"
721 : : */
722 : : static void
116 rhaas@postgresql.org 723 :GNC 1 : help(const char *progname)
724 : : {
725 : 1 : printf(_("%s reconstructs full backups from incrementals.\n\n"), progname);
726 : 1 : printf(_("Usage:\n"));
727 : 1 : printf(_(" %s [OPTION]... DIRECTORY...\n"), progname);
728 : 1 : printf(_("\nOptions:\n"));
729 : 1 : printf(_(" -d, --debug generate lots of debugging output\n"));
730 : 1 : printf(_(" -n, --dry-run don't actually do anything\n"));
731 : 1 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
732 : 1 : printf(_(" -o, --output output directory\n"));
110 michael@paquier.xyz 733 : 1 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
734 : : " relocate tablespace in OLDDIR to NEWDIR\n"));
116 rhaas@postgresql.org 735 : 1 : printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
736 : : " use algorithm for manifest checksums\n"));
737 : 1 : printf(_(" --no-manifest suppress generation of backup manifest\n"));
738 : 1 : printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
9 tomas.vondra@postgre 739 : 1 : printf(_(" --clone clone (reflink) instead of copying files\n"));
740 : 1 : printf(_(" --copy-file-range copy using copy_file_range() syscall\n"));
116 rhaas@postgresql.org 741 : 1 : printf(_(" -?, --help show this help, then exit\n"));
742 : :
743 : 1 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
744 : 1 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
745 : 1 : }
746 : :
747 : : /*
748 : : * Try to parse a string as a non-zero OID without leading zeroes.
749 : : *
750 : : * If it works, return true and set *result to the answer, else return false.
751 : : */
752 : : static bool
116 rhaas@postgresql.org 753 :UNC 0 : parse_oid(char *s, Oid *result)
754 : : {
755 : : Oid oid;
756 : : char *ep;
757 : :
758 : 0 : errno = 0;
759 : 0 : oid = strtoul(s, &ep, 10);
760 [ # # # # : 0 : if (errno != 0 || *ep != '\0' || oid < 1 || oid > PG_UINT32_MAX)
# # ]
761 : 0 : return false;
762 : :
763 : 0 : *result = oid;
764 : 0 : return true;
765 : : }
766 : :
767 : : /*
768 : : * Copy files from the input directory to the output directory, reconstructing
769 : : * full files from incremental files as required.
770 : : *
771 : : * If processing is a user-defined tablespace, the tsoid should be the OID
772 : : * of that tablespace and input_directory and output_directory should be the
773 : : * toplevel input and output directories for that tablespace. Otherwise,
774 : : * tsoid should be InvalidOid and input_directory and output_directory should
775 : : * be the main input and output directories.
776 : : *
777 : : * relative_path is the path beneath the given input and output directories
778 : : * that we are currently processing. If NULL, it indicates that we're
779 : : * processing the input and output directories themselves.
780 : : *
781 : : * n_prior_backups is the number of prior backups that we have available.
782 : : * This doesn't count the very last backup, which is referenced by
783 : : * output_directory, just the older ones. prior_backup_dirs is an array of
784 : : * the locations of those previous backups.
785 : : */
786 : : static void
116 rhaas@postgresql.org 787 :GNC 272 : process_directory_recursively(Oid tsoid,
788 : : char *input_directory,
789 : : char *output_directory,
790 : : char *relative_path,
791 : : int n_prior_backups,
792 : : char **prior_backup_dirs,
793 : : manifest_data **manifests,
794 : : manifest_writer *mwriter,
795 : : cb_options *opt)
796 : : {
797 : : char ifulldir[MAXPGPATH];
798 : : char ofulldir[MAXPGPATH];
799 : : char manifest_prefix[MAXPGPATH];
800 : : DIR *dir;
801 : : struct dirent *de;
802 : : bool is_pg_tblspc;
803 : : bool is_pg_wal;
804 : 272 : manifest_data *latest_manifest = manifests[n_prior_backups];
805 : : pg_checksum_type checksum_type;
806 : :
807 : : /*
808 : : * pg_tblspc and pg_wal are special cases, so detect those here.
809 : : *
810 : : * pg_tblspc is only special at the top level, but subdirectories of
811 : : * pg_wal are just as special as the top level directory.
812 : : *
813 : : * Since incremental backup does not exist in pre-v10 versions, we don't
814 : : * have to worry about the old pg_xlog naming.
815 : : */
816 [ + - + + ]: 534 : is_pg_tblspc = !OidIsValid(tsoid) && relative_path != NULL &&
817 [ + + ]: 262 : strcmp(relative_path, "pg_tblspc") == 0;
818 [ + - + + ]: 534 : is_pg_wal = !OidIsValid(tsoid) && relative_path != NULL &&
819 [ + + ]: 262 : (strcmp(relative_path, "pg_wal") == 0 ||
820 [ + + ]: 252 : strncmp(relative_path, "pg_wal/", 7) == 0);
821 : :
822 : : /*
823 : : * If we're under pg_wal, then we don't need checksums, because these
824 : : * files aren't included in the backup manifest. Otherwise use whatever
825 : : * type of checksum is configured.
826 : : */
827 [ + + ]: 272 : if (!is_pg_wal)
828 : 242 : checksum_type = opt->manifest_checksums;
829 : : else
830 : 30 : checksum_type = CHECKSUM_TYPE_NONE;
831 : :
832 : : /*
833 : : * Append the relative path to the input and output directories, and
834 : : * figure out the appropriate prefix to add to files in this directory
835 : : * when looking them up in a backup manifest.
836 : : */
837 [ + + ]: 272 : if (relative_path == NULL)
838 : : {
94 839 : 10 : strlcpy(ifulldir, input_directory, MAXPGPATH);
840 : 10 : strlcpy(ofulldir, output_directory, MAXPGPATH);
116 841 [ - + ]: 10 : if (OidIsValid(tsoid))
116 rhaas@postgresql.org 842 :UNC 0 : snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/", tsoid);
843 : : else
116 rhaas@postgresql.org 844 :GNC 10 : manifest_prefix[0] = '\0';
845 : : }
846 : : else
847 : : {
848 : 262 : snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
849 : : relative_path);
850 : 262 : snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
851 : : relative_path);
852 [ - + ]: 262 : if (OidIsValid(tsoid))
116 rhaas@postgresql.org 853 :UNC 0 : snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/",
854 : : tsoid, relative_path);
855 : : else
116 rhaas@postgresql.org 856 :GNC 262 : snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
857 : : }
858 : :
859 : : /*
860 : : * Toplevel output directories have already been created by the time this
861 : : * function is called, but any subdirectories are our responsibility.
862 : : */
863 [ + + ]: 272 : if (relative_path != NULL)
864 : : {
865 [ - + ]: 262 : if (opt->dry_run)
116 rhaas@postgresql.org 866 [ # # ]:UNC 0 : pg_log_debug("would create directory \"%s\"", ofulldir);
867 : : else
868 : : {
116 rhaas@postgresql.org 869 [ + + ]:GNC 262 : pg_log_debug("creating directory \"%s\"", ofulldir);
870 [ - + ]: 262 : if (mkdir(ofulldir, pg_dir_create_mode) == -1)
116 rhaas@postgresql.org 871 :UNC 0 : pg_fatal("could not create directory \"%s\": %m", ofulldir);
872 : : }
873 : : }
874 : :
875 : : /* It's time to scan the directory. */
116 rhaas@postgresql.org 876 [ - + ]:GNC 272 : if ((dir = opendir(ifulldir)) == NULL)
116 rhaas@postgresql.org 877 :UNC 0 : pg_fatal("could not open directory \"%s\": %m", ifulldir);
116 rhaas@postgresql.org 878 [ + + ]:GNC 11382 : while (errno = 0, (de = readdir(dir)) != NULL)
879 : : {
880 : : PGFileType type;
881 : : char ifullpath[MAXPGPATH];
882 : : char ofullpath[MAXPGPATH];
883 : : char manifest_path[MAXPGPATH];
884 : 11110 : Oid oid = InvalidOid;
885 : 11110 : int checksum_length = 0;
886 : 11110 : uint8 *checksum_payload = NULL;
887 : : pg_checksum_context checksum_ctx;
888 : :
889 : : /* Ignore "." and ".." entries. */
890 [ + + ]: 11110 : if (strcmp(de->d_name, ".") == 0 ||
891 [ + + ]: 10838 : strcmp(de->d_name, "..") == 0)
892 : 826 : continue;
893 : :
894 : : /* Construct input path. */
895 : 10566 : snprintf(ifullpath, MAXPGPATH, "%s/%s", ifulldir, de->d_name);
896 : :
897 : : /* Figure out what kind of directory entry this is. */
898 : 10566 : type = get_dirent_type(ifullpath, de, false, PG_LOG_ERROR);
899 [ - + ]: 10566 : if (type == PGFILETYPE_ERROR)
116 rhaas@postgresql.org 900 :UNC 0 : exit(1);
901 : :
902 : : /*
903 : : * If we're processing pg_tblspc, then check whether the filename
904 : : * looks like it could be a tablespace OID. If so, and if the
905 : : * directory entry is a symbolic link or a directory, skip it.
906 : : *
907 : : * Our goal here is to ignore anything that would have been considered
908 : : * by scan_for_existing_tablespaces to be a tablespace.
909 : : */
116 rhaas@postgresql.org 910 [ - + - - :GNC 10566 : if (is_pg_tblspc && parse_oid(de->d_name, &oid) &&
- - ]
116 rhaas@postgresql.org 911 [ # # ]:UNC 0 : (type == PGFILETYPE_LNK || type == PGFILETYPE_DIR))
912 : 0 : continue;
913 : :
914 : : /* If it's a directory, recurse. */
116 rhaas@postgresql.org 915 [ + + ]:GNC 10566 : if (type == PGFILETYPE_DIR)
916 : 262 : {
917 : : char new_relative_path[MAXPGPATH];
918 : :
919 : : /* Append new pathname component to relative path. */
920 [ + + ]: 262 : if (relative_path == NULL)
94 921 : 170 : strlcpy(new_relative_path, de->d_name, MAXPGPATH);
922 : : else
116 923 : 92 : snprintf(new_relative_path, MAXPGPATH, "%s/%s", relative_path,
924 : 92 : de->d_name);
925 : :
926 : : /* And recurse. */
927 : 262 : process_directory_recursively(tsoid,
928 : : input_directory, output_directory,
929 : : new_relative_path,
930 : : n_prior_backups, prior_backup_dirs,
931 : : manifests, mwriter, opt);
932 : 262 : continue;
933 : : }
934 : :
935 : : /* Skip anything that's not a regular file. */
936 [ - + ]: 10304 : if (type != PGFILETYPE_REG)
937 : : {
116 rhaas@postgresql.org 938 [ # # ]:UNC 0 : if (type == PGFILETYPE_LNK)
939 : 0 : pg_log_warning("skipping symbolic link \"%s\"", ifullpath);
940 : : else
941 : 0 : pg_log_warning("skipping special file \"%s\"", ifullpath);
942 : 0 : continue;
943 : : }
944 : :
945 : : /*
946 : : * Skip the backup_label and backup_manifest files; they require
947 : : * special handling and are handled elsewhere.
948 : : */
116 rhaas@postgresql.org 949 [ + + ]:GNC 10304 : if (relative_path == NULL &&
950 [ + + ]: 71 : (strcmp(de->d_name, "backup_label") == 0 ||
951 [ + + ]: 61 : strcmp(de->d_name, "backup_manifest") == 0))
952 : 20 : continue;
953 : :
954 : : /*
955 : : * If it's an incremental file, hand it off to the reconstruction
956 : : * code, which will figure out what to do.
957 : : */
958 [ + + ]: 10284 : if (strncmp(de->d_name, INCREMENTAL_PREFIX,
959 : : INCREMENTAL_PREFIX_LENGTH) == 0)
960 : : {
961 : : /* Output path should not include "INCREMENTAL." prefix. */
962 : 4651 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
963 : 4651 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
964 : :
965 : :
966 : : /* Manifest path likewise omits incremental prefix. */
967 : 4651 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
968 : 4651 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
969 : :
970 : : /* Reconstruction logic will do the rest. */
971 : 4651 : reconstruct_from_incremental_file(ifullpath, ofullpath,
972 : : relative_path,
973 : 4651 : de->d_name + INCREMENTAL_PREFIX_LENGTH,
974 : : n_prior_backups,
975 : : prior_backup_dirs,
976 : : manifests,
977 : : manifest_path,
978 : : checksum_type,
979 : : &checksum_length,
980 : : &checksum_payload,
981 : : opt->copy_method,
982 : 4651 : opt->debug,
983 : 4651 : opt->dry_run);
984 : : }
985 : : else
986 : : {
987 : : /* Construct the path that the backup_manifest will use. */
988 : 5633 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
989 : 5633 : de->d_name);
990 : :
991 : : /*
992 : : * It's not an incremental file, so we need to copy the entire
993 : : * file to the output directory.
994 : : *
995 : : * If a checksum of the required type already exists in the
996 : : * backup_manifest for the final input directory, we can save some
997 : : * work by reusing that checksum instead of computing a new one.
998 : : */
999 [ + + + - ]: 5633 : if (checksum_type != CHECKSUM_TYPE_NONE &&
1000 : : latest_manifest != NULL)
1001 : : {
1002 : : manifest_file *mfile;
1003 : :
1004 : 3693 : mfile = manifest_files_lookup(latest_manifest->files,
1005 : : manifest_path);
1006 [ - + ]: 3693 : if (mfile == NULL)
1007 : : {
1008 : : char *bmpath;
1009 : :
1010 : : /*
1011 : : * The directory is out of sync with the backup_manifest,
1012 : : * so emit a warning.
1013 : : */
116 rhaas@postgresql.org 1014 :UNC 0 : bmpath = psprintf("%s/%s", input_directory,
1015 : : "backup_manifest");
1016 : 0 : pg_log_warning("\"%s\" contains no entry for \"%s\"",
1017 : : bmpath, manifest_path);
1018 : 0 : pfree(bmpath);
1019 : : }
116 rhaas@postgresql.org 1020 [ + + ]:GNC 3693 : else if (mfile->checksum_type == checksum_type)
1021 : : {
1022 : 2728 : checksum_length = mfile->checksum_length;
1023 : 2728 : checksum_payload = mfile->checksum_payload;
1024 : : }
1025 : : }
1026 : :
1027 : : /*
1028 : : * If we're reusing a checksum, then we don't need copy_file() to
1029 : : * compute one for us, but otherwise, it needs to compute whatever
1030 : : * type of checksum we need.
1031 : : */
1032 [ + + ]: 5633 : if (checksum_length != 0)
1033 : 2728 : pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_NONE);
1034 : : else
1035 : 2905 : pg_checksum_init(&checksum_ctx, checksum_type);
1036 : :
1037 : : /* Actually copy the file. */
1038 : 5633 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
9 tomas.vondra@postgre 1039 : 5633 : copy_file(ifullpath, ofullpath, &checksum_ctx,
1040 : 5633 : opt->copy_method, opt->dry_run);
1041 : :
1042 : : /*
1043 : : * If copy_file() performed a checksum calculation for us, then
1044 : : * save the results (except in dry-run mode, when there's no
1045 : : * point).
1046 : : */
116 rhaas@postgresql.org 1047 [ + + + - ]: 5633 : if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
1048 : : {
1049 : 965 : checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
1050 : 965 : checksum_length = pg_checksum_final(&checksum_ctx,
1051 : : checksum_payload);
1052 : : }
1053 : : }
1054 : :
1055 : : /* Generate manifest entry, if needed. */
1056 [ + + ]: 10284 : if (mwriter != NULL)
1057 : : {
1058 : : struct stat sb;
1059 : :
1060 : : /*
1061 : : * In order to generate a manifest entry, we need the file size
1062 : : * and mtime. We have no way to know the correct mtime except to
1063 : : * stat() the file, so just do that and get the size as well.
1064 : : *
1065 : : * If we didn't need the mtime here, we could try to obtain the
1066 : : * file size from the reconstruction or file copy process above,
1067 : : * although that is actually not convenient in all cases. If we
1068 : : * write the file ourselves then clearly we can keep a count of
1069 : : * bytes, but if we use something like CopyFile() then it's
1070 : : * trickier. Since we have to stat() anyway to get the mtime,
1071 : : * there's no point in worrying about it.
1072 : : */
1073 [ - + ]: 9318 : if (stat(ofullpath, &sb) < 0)
116 rhaas@postgresql.org 1074 :UNC 0 : pg_fatal("could not stat file \"%s\": %m", ofullpath);
1075 : :
1076 : : /* OK, now do the work. */
116 rhaas@postgresql.org 1077 :GNC 9318 : add_file_to_manifest(mwriter, manifest_path,
1078 : 9318 : sb.st_size, sb.st_mtime,
1079 : : checksum_type, checksum_length,
1080 : : checksum_payload);
1081 : : }
1082 : :
1083 : : /* Avoid leaking memory. */
1084 [ + + ]: 10284 : if (checksum_payload != NULL)
1085 : 8344 : pfree(checksum_payload);
1086 : : }
1087 : :
1088 : 272 : closedir(dir);
1089 : 272 : }
1090 : :
1091 : : /*
1092 : : * Read the version number from PG_VERSION and convert it to the usual server
1093 : : * version number format. (e.g. If PG_VERSION contains "14\n" this function
1094 : : * will return 140000)
1095 : : */
1096 : : static int
1097 : 17 : read_pg_version_file(char *directory)
1098 : : {
1099 : : char filename[MAXPGPATH];
1100 : : StringInfoData buf;
1101 : : int fd;
1102 : : int version;
1103 : : char *ep;
1104 : :
1105 : : /* Construct pathname. */
1106 : 17 : snprintf(filename, MAXPGPATH, "%s/PG_VERSION", directory);
1107 : :
1108 : : /* Open file. */
1109 [ - + ]: 17 : if ((fd = open(filename, O_RDONLY, 0)) < 0)
116 rhaas@postgresql.org 1110 :UNC 0 : pg_fatal("could not open file \"%s\": %m", filename);
1111 : :
1112 : : /* Read into memory. Length limit of 128 should be more than generous. */
116 rhaas@postgresql.org 1113 :GNC 17 : initStringInfo(&buf);
1114 : 17 : slurp_file(fd, filename, &buf, 128);
1115 : :
1116 : : /* Close the file. */
1117 [ - + ]: 17 : if (close(fd) != 0)
116 rhaas@postgresql.org 1118 :UNC 0 : pg_fatal("could not close \"%s\": %m", filename);
1119 : :
1120 : : /* Convert to integer. */
116 rhaas@postgresql.org 1121 :GNC 17 : errno = 0;
1122 : 17 : version = strtoul(buf.data, &ep, 10);
1123 [ + - - + ]: 17 : if (errno != 0 || *ep != '\n')
1124 : : {
1125 : : /*
1126 : : * Incremental backup is not relevant to very old server versions that
1127 : : * used multi-part version number (e.g. 9.6, or 8.4). So if we see
1128 : : * what looks like the beginning of such a version number, just bail
1129 : : * out.
1130 : : */
116 rhaas@postgresql.org 1131 [ # # # # ]:UNC 0 : if (version < 10 && *ep == '.')
1132 : 0 : pg_fatal("%s: server version too old\n", filename);
1133 : 0 : pg_fatal("%s: could not parse version number\n", filename);
1134 : : }
1135 : :
1136 : : /* Debugging output. */
116 rhaas@postgresql.org 1137 [ + + ]:GNC 17 : pg_log_debug("read server version %d from \"%s\"", version, filename);
1138 : :
1139 : : /* Release memory and return result. */
1140 : 17 : pfree(buf.data);
1141 : 17 : return version * 10000;
1142 : : }
1143 : :
1144 : : /*
1145 : : * Add a directory to the list of output directories to clean up.
1146 : : */
1147 : : static void
1148 : 10 : remember_to_cleanup_directory(char *target_path, bool rmtopdir)
1149 : : {
1150 : 10 : cb_cleanup_dir *dir = pg_malloc(sizeof(cb_cleanup_dir));
1151 : :
1152 : 10 : dir->target_path = target_path;
1153 : 10 : dir->rmtopdir = rmtopdir;
1154 : 10 : dir->next = cleanup_dir_list;
1155 : 10 : cleanup_dir_list = dir;
1156 : 10 : }
1157 : :
1158 : : /*
1159 : : * Empty out the list of directories scheduled for cleanup a exit.
1160 : : *
1161 : : * We want to remove the output directories only on a failure, so call this
1162 : : * function when we know that the operation has succeeded.
1163 : : *
1164 : : * Since we only expect this to be called when we're about to exit, we could
1165 : : * just set cleanup_dir_list to NULL and be done with it, but we free the
1166 : : * memory to be tidy.
1167 : : */
1168 : : static void
1169 : 10 : reset_directory_cleanup_list(void)
1170 : : {
1171 [ + + ]: 20 : while (cleanup_dir_list != NULL)
1172 : : {
1173 : 10 : cb_cleanup_dir *dir = cleanup_dir_list;
1174 : :
1175 : 10 : cleanup_dir_list = cleanup_dir_list->next;
1176 : 10 : pfree(dir);
1177 : : }
1178 : 10 : }
1179 : :
1180 : : /*
1181 : : * Scan the pg_tblspc directory of the final input backup to get a canonical
1182 : : * list of what tablespaces are part of the backup.
1183 : : *
1184 : : * 'pathname' should be the path to the toplevel backup directory for the
1185 : : * final backup in the backup chain.
1186 : : */
1187 : : static cb_tablespace *
1188 : 10 : scan_for_existing_tablespaces(char *pathname, cb_options *opt)
1189 : : {
1190 : : char pg_tblspc[MAXPGPATH];
1191 : : DIR *dir;
1192 : : struct dirent *de;
1193 : 10 : cb_tablespace *tslist = NULL;
1194 : :
1195 : 10 : snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pathname);
1196 [ + + ]: 10 : pg_log_debug("scanning \"%s\"", pg_tblspc);
1197 : :
1198 [ - + ]: 10 : if ((dir = opendir(pg_tblspc)) == NULL)
94 rhaas@postgresql.org 1199 :UNC 0 : pg_fatal("could not open directory \"%s\": %m", pg_tblspc);
1200 : :
116 rhaas@postgresql.org 1201 [ + + ]:GNC 30 : while (errno = 0, (de = readdir(dir)) != NULL)
1202 : : {
1203 : : Oid oid;
1204 : : char tblspcdir[MAXPGPATH];
1205 : : char link_target[MAXPGPATH];
1206 : : int link_length;
1207 : : cb_tablespace *ts;
1208 : : cb_tablespace *otherts;
1209 : : PGFileType type;
1210 : :
1211 : : /* Silently ignore "." and ".." entries. */
1212 [ + + + - ]: 20 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1213 : 20 : continue;
1214 : :
1215 : : /* Construct full pathname. */
116 rhaas@postgresql.org 1216 :UNC 0 : snprintf(tblspcdir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1217 : :
1218 : : /* Ignore any file name that doesn't look like a proper OID. */
1219 [ # # ]: 0 : if (!parse_oid(de->d_name, &oid))
1220 : : {
1221 [ # # ]: 0 : pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
1222 : : tblspcdir);
1223 : 0 : continue;
1224 : : }
1225 : :
1226 : : /* Only symbolic links and directories are tablespaces. */
1227 : 0 : type = get_dirent_type(tblspcdir, de, false, PG_LOG_ERROR);
1228 [ # # ]: 0 : if (type == PGFILETYPE_ERROR)
1229 : 0 : exit(1);
1230 [ # # # # ]: 0 : if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
1231 : : {
1232 [ # # ]: 0 : pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
1233 : : tblspcdir);
1234 : 0 : continue;
1235 : : }
1236 : :
1237 : : /* Create a new tablespace object. */
1238 : 0 : ts = pg_malloc0(sizeof(cb_tablespace));
1239 : 0 : ts->oid = oid;
1240 : :
1241 : : /*
1242 : : * If it's a link, it's not an in-place tablespace. Otherwise, it must
1243 : : * be a directory, and thus an in-place tablespace.
1244 : : */
1245 [ # # ]: 0 : if (type == PGFILETYPE_LNK)
1246 : : {
1247 : : cb_tablespace_mapping *tsmap;
1248 : :
1249 : : /* Read the link target. */
1250 : 0 : link_length = readlink(tblspcdir, link_target, sizeof(link_target));
1251 [ # # ]: 0 : if (link_length < 0)
1252 : 0 : pg_fatal("could not read symbolic link \"%s\": %m",
1253 : : tblspcdir);
1254 [ # # ]: 0 : if (link_length >= sizeof(link_target))
1255 : 0 : pg_fatal("symbolic link \"%s\" is too long", tblspcdir);
1256 : 0 : link_target[link_length] = '\0';
1257 [ # # ]: 0 : if (!is_absolute_path(link_target))
1258 : 0 : pg_fatal("symbolic link \"%s\" is relative", tblspcdir);
1259 : :
1260 : : /* Canonicalize the link target. */
1261 : 0 : canonicalize_path(link_target);
1262 : :
1263 : : /*
1264 : : * Find the corresponding tablespace mapping and copy the relevant
1265 : : * details into the new tablespace entry.
1266 : : */
1267 [ # # ]: 0 : for (tsmap = opt->tsmappings; tsmap != NULL; tsmap = tsmap->next)
1268 : : {
1269 [ # # ]: 0 : if (strcmp(tsmap->old_dir, link_target) == 0)
1270 : : {
94 1271 : 0 : strlcpy(ts->old_dir, tsmap->old_dir, MAXPGPATH);
1272 : 0 : strlcpy(ts->new_dir, tsmap->new_dir, MAXPGPATH);
116 1273 : 0 : ts->in_place = false;
1274 : 0 : break;
1275 : : }
1276 : : }
1277 : :
1278 : : /* Every non-in-place tablespace must be mapped. */
1279 [ # # ]: 0 : if (tsmap == NULL)
1280 : 0 : pg_fatal("tablespace at \"%s\" has no tablespace mapping",
1281 : : link_target);
1282 : : }
1283 : : else
1284 : : {
1285 : : /*
1286 : : * For an in-place tablespace, there's no separate directory, so
1287 : : * we just record the paths within the data directories.
1288 : : */
1289 : 0 : snprintf(ts->old_dir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
115 1290 : 0 : snprintf(ts->new_dir, MAXPGPATH, "%s/pg_tblspc/%s", opt->output,
116 1291 : 0 : de->d_name);
1292 : 0 : ts->in_place = true;
1293 : : }
1294 : :
1295 : : /* Tablespaces should not share a directory. */
1296 [ # # ]: 0 : for (otherts = tslist; otherts != NULL; otherts = otherts->next)
1297 [ # # ]: 0 : if (strcmp(ts->new_dir, otherts->new_dir) == 0)
1298 : 0 : pg_fatal("tablespaces with OIDs %u and %u both point at \"%s\"",
1299 : : otherts->oid, oid, ts->new_dir);
1300 : :
1301 : : /* Add this tablespace to the list. */
1302 : 0 : ts->next = tslist;
1303 : 0 : tslist = ts;
1304 : : }
1305 : :
94 rhaas@postgresql.org 1306 [ - + ]:GNC 10 : if (closedir(dir) != 0)
94 rhaas@postgresql.org 1307 :UNC 0 : pg_fatal("could not close directory \"%s\": %m", pg_tblspc);
1308 : :
116 rhaas@postgresql.org 1309 :GNC 10 : return tslist;
1310 : : }
1311 : :
1312 : : /*
1313 : : * Read a file into a StringInfo.
1314 : : *
1315 : : * fd is used for the actual file I/O, filename for error reporting purposes.
1316 : : * A file longer than maxlen is a fatal error.
1317 : : */
1318 : : static void
1319 : 48 : slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
1320 : : {
1321 : : struct stat st;
1322 : : ssize_t rb;
1323 : :
1324 : : /* Check file size, and complain if it's too large. */
1325 [ - + ]: 48 : if (fstat(fd, &st) != 0)
116 rhaas@postgresql.org 1326 :UNC 0 : pg_fatal("could not stat \"%s\": %m", filename);
116 rhaas@postgresql.org 1327 [ - + ]:GNC 48 : if (st.st_size > maxlen)
116 rhaas@postgresql.org 1328 :UNC 0 : pg_fatal("file \"%s\" is too large", filename);
1329 : :
1330 : : /* Make sure we have enough space. */
116 rhaas@postgresql.org 1331 :GNC 48 : enlargeStringInfo(buf, st.st_size);
1332 : :
1333 : : /* Read the data. */
1334 : 48 : rb = read(fd, &buf->data[buf->len], st.st_size);
1335 : :
1336 : : /*
1337 : : * We don't expect any concurrent changes, so we should read exactly the
1338 : : * expected number of bytes.
1339 : : */
1340 [ - + ]: 48 : if (rb != st.st_size)
1341 : : {
116 rhaas@postgresql.org 1342 [ # # ]:UNC 0 : if (rb < 0)
1343 : 0 : pg_fatal("could not read file \"%s\": %m", filename);
1344 : : else
32 peter@eisentraut.org 1345 : 0 : pg_fatal("could not read file \"%s\": read only %zd of %lld bytes",
1346 : : filename, rb, (long long int) st.st_size);
1347 : : }
1348 : :
1349 : : /* Adjust buffer length for new data and restore trailing-\0 invariant */
116 rhaas@postgresql.org 1350 :GNC 48 : buf->len += rb;
1351 : 48 : buf->data[buf->len] = '\0';
1352 : 48 : }
|
