Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * reconstruct.c
4 : : * Reconstruct full file from incremental file and backup chain.
5 : : *
6 : : * Copyright (c) 2017-2024, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/bin/pg_combinebackup/reconstruct.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres_fe.h"
14 : :
15 : : #include <unistd.h>
16 : :
17 : : #include "backup/basebackup_incremental.h"
18 : : #include "common/file_perm.h"
19 : : #include "common/logging.h"
20 : : #include "copy_file.h"
21 : : #include "lib/stringinfo.h"
22 : : #include "reconstruct.h"
23 : : #include "storage/block.h"
24 : :
25 : : /*
26 : : * An rfile stores the data that we need in order to be able to use some file
27 : : * on disk for reconstruction. For any given output file, we create one rfile
28 : : * per backup that we need to consult when we constructing that output file.
29 : : *
30 : : * If we find a full version of the file in the backup chain, then only
31 : : * filename and fd are initialized; the remaining fields are 0 or NULL.
32 : : * For an incremental file, header_length, num_blocks, relative_block_numbers,
33 : : * and truncation_block_length are also set.
34 : : *
35 : : * num_blocks_read and highest_offset_read always start out as 0.
36 : : */
37 : : typedef struct rfile
38 : : {
39 : : char *filename;
40 : : int fd;
41 : : size_t header_length;
42 : : unsigned num_blocks;
43 : : BlockNumber *relative_block_numbers;
44 : : unsigned truncation_block_length;
45 : : unsigned num_blocks_read;
46 : : off_t highest_offset_read;
47 : : } rfile;
48 : :
49 : : static void debug_reconstruction(int n_source,
50 : : rfile **sources,
51 : : bool dry_run);
52 : : static unsigned find_reconstructed_block_length(rfile *s);
53 : : static rfile *make_incremental_rfile(char *filename);
54 : : static rfile *make_rfile(char *filename, bool missing_ok);
55 : : static void write_reconstructed_file(char *input_filename,
56 : : char *output_filename,
57 : : unsigned block_length,
58 : : rfile **sourcemap,
59 : : off_t *offsetmap,
60 : : pg_checksum_context *checksum_ctx,
61 : : CopyMethod copy_method,
62 : : bool debug,
63 : : bool dry_run);
64 : : static void read_bytes(rfile *rf, void *buffer, unsigned length);
65 : : static void write_block(int wfd, char *output_filename,
66 : : uint8 *buffer,
67 : : pg_checksum_context *checksum_ctx);
68 : : static void read_block(rfile *s, off_t off, uint8 *buffer);
69 : :
70 : : /*
71 : : * Reconstruct a full file from an incremental file and a chain of prior
72 : : * backups.
73 : : *
74 : : * input_filename should be the path to the incremental file, and
75 : : * output_filename should be the path where the reconstructed file is to be
76 : : * written.
77 : : *
78 : : * relative_path should be the relative path to the directory containing this
79 : : * file. bare_file_name should be the name of the file within that directory,
80 : : * without "INCREMENTAL.".
81 : : *
82 : : * n_prior_backups is the number of prior backups, and prior_backup_dirs is
83 : : * an array of pathnames where those backups can be found.
84 : : */
85 : : void
116 rhaas@postgresql.org 86 :GNC 4651 : reconstruct_from_incremental_file(char *input_filename,
87 : : char *output_filename,
88 : : char *relative_path,
89 : : char *bare_file_name,
90 : : int n_prior_backups,
91 : : char **prior_backup_dirs,
92 : : manifest_data **manifests,
93 : : char *manifest_path,
94 : : pg_checksum_type checksum_type,
95 : : int *checksum_length,
96 : : uint8 **checksum_payload,
97 : : CopyMethod copy_method,
98 : : bool debug,
99 : : bool dry_run)
100 : : {
101 : : rfile **source;
102 : 4651 : rfile *latest_source = NULL;
103 : : rfile **sourcemap;
104 : : off_t *offsetmap;
105 : : unsigned block_length;
106 : : unsigned i;
107 : 4651 : unsigned sidx = n_prior_backups;
108 : 4651 : bool full_copy_possible = true;
109 : 4651 : int copy_source_index = -1;
110 : 4651 : rfile *copy_source = NULL;
111 : : pg_checksum_context checksum_ctx;
112 : :
113 : : /*
114 : : * Every block must come either from the latest version of the file or
115 : : * from one of the prior backups.
116 : : */
117 : 4651 : source = pg_malloc0(sizeof(rfile *) * (1 + n_prior_backups));
118 : :
119 : : /*
120 : : * Use the information from the latest incremental file to figure out how
121 : : * long the reconstructed file should be.
122 : : */
123 : 4651 : latest_source = make_incremental_rfile(input_filename);
124 : 4651 : source[n_prior_backups] = latest_source;
125 : 4651 : block_length = find_reconstructed_block_length(latest_source);
126 : :
127 : : /*
128 : : * For each block in the output file, we need to know from which file we
129 : : * need to obtain it and at what offset in that file it's stored.
130 : : * sourcemap gives us the first of these things, and offsetmap the latter.
131 : : */
132 : 4651 : sourcemap = pg_malloc0(sizeof(rfile *) * block_length);
133 : 4651 : offsetmap = pg_malloc0(sizeof(off_t) * block_length);
134 : :
135 : : /*
136 : : * Every block that is present in the newest incremental file should be
137 : : * sourced from that file. If it precedes the truncation_block_length,
138 : : * it's a block that we would otherwise have had to find in an older
139 : : * backup and thus reduces the number of blocks remaining to be found by
140 : : * one; otherwise, it's an extra block that needs to be included in the
141 : : * output but would not have needed to be found in an older backup if it
142 : : * had not been present.
143 : : */
144 [ + + ]: 4686 : for (i = 0; i < latest_source->num_blocks; ++i)
145 : : {
146 : 35 : BlockNumber b = latest_source->relative_block_numbers[i];
147 : :
148 [ - + ]: 35 : Assert(b < block_length);
149 : 35 : sourcemap[b] = latest_source;
150 : 35 : offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
151 : :
152 : : /*
153 : : * A full copy of a file from an earlier backup is only possible if no
154 : : * blocks are needed from any later incremental file.
155 : : */
156 : 35 : full_copy_possible = false;
157 : : }
158 : :
159 : : while (1)
160 : 1329 : {
161 : : char source_filename[MAXPGPATH];
162 : : rfile *s;
163 : :
164 : : /*
165 : : * Move to the next backup in the chain. If there are no more, then
166 : : * we're done.
167 : : */
168 [ - + ]: 5980 : if (sidx == 0)
116 rhaas@postgresql.org 169 :UNC 0 : break;
116 rhaas@postgresql.org 170 :GNC 5980 : --sidx;
171 : :
172 : : /*
173 : : * Look for the full file in the previous backup. If not found, then
174 : : * look for an incremental file instead.
175 : : */
176 : 5980 : snprintf(source_filename, MAXPGPATH, "%s/%s/%s",
177 : 5980 : prior_backup_dirs[sidx], relative_path, bare_file_name);
178 [ + + ]: 5980 : if ((s = make_rfile(source_filename, true)) == NULL)
179 : : {
180 : 1329 : snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s",
181 : 1329 : prior_backup_dirs[sidx], relative_path, bare_file_name);
182 : 1329 : s = make_incremental_rfile(source_filename);
183 : : }
184 : 5980 : source[sidx] = s;
185 : :
186 : : /*
187 : : * If s->header_length == 0, then this is a full file; otherwise, it's
188 : : * an incremental file.
189 : : */
190 [ + + ]: 5980 : if (s->header_length == 0)
191 : : {
192 : : struct stat sb;
193 : : BlockNumber b;
194 : : BlockNumber blocklength;
195 : :
196 : : /* We need to know the length of the file. */
197 [ - + ]: 4651 : if (fstat(s->fd, &sb) < 0)
116 rhaas@postgresql.org 198 :UNC 0 : pg_fatal("could not stat \"%s\": %m", s->filename);
199 : :
200 : : /*
201 : : * Since we found a full file, source all blocks from it that
202 : : * exist in the file.
203 : : *
204 : : * Note that there may be blocks that don't exist either in this
205 : : * file or in any incremental file but that precede
206 : : * truncation_block_length. These are, presumably, zero-filled
207 : : * blocks that result from the server extending the file but
208 : : * taking no action on those blocks that generated any WAL.
209 : : *
210 : : * Sadly, we have no way of validating that this is really what
211 : : * happened, and neither does the server. From it's perspective,
212 : : * an unmodified block that contains data looks exactly the same
213 : : * as a zero-filled block that never had any data: either way,
214 : : * it's not mentioned in any WAL summary and the server has no
215 : : * reason to read it. From our perspective, all we know is that
216 : : * nobody had a reason to back up the block. That certainly means
217 : : * that the block didn't exist at the time of the full backup, but
218 : : * the supposition that it was all zeroes at the time of every
219 : : * later backup is one that we can't validate.
220 : : */
116 rhaas@postgresql.org 221 :GNC 4651 : blocklength = sb.st_size / BLCKSZ;
222 [ + + ]: 22323 : for (b = 0; b < latest_source->truncation_block_length; ++b)
223 : : {
224 [ + + + - ]: 17672 : if (sourcemap[b] == NULL && b < blocklength)
225 : : {
226 : 17637 : sourcemap[b] = s;
227 : 17637 : offsetmap[b] = b * BLCKSZ;
228 : : }
229 : : }
230 : :
231 : : /*
232 : : * If a full copy looks possible, check whether the resulting file
233 : : * should be exactly as long as the source file is. If so, a full
234 : : * copy is acceptable, otherwise not.
235 : : */
236 [ + + ]: 4651 : if (full_copy_possible)
237 : : {
238 : : uint64 expected_length;
239 : :
240 : 4628 : expected_length =
241 : 4628 : (uint64) latest_source->truncation_block_length;
242 : 4628 : expected_length *= BLCKSZ;
243 [ + - ]: 4628 : if (expected_length == sb.st_size)
244 : : {
245 : 4628 : copy_source = s;
246 : 4628 : copy_source_index = sidx;
247 : : }
248 : : }
249 : :
250 : : /* We don't need to consider any further sources. */
251 : 4651 : break;
252 : : }
253 : :
254 : : /*
255 : : * Since we found another incremental file, source all blocks from it
256 : : * that we need but don't yet have.
257 : : */
258 [ - + ]: 1329 : for (i = 0; i < s->num_blocks; ++i)
259 : : {
116 rhaas@postgresql.org 260 :UNC 0 : BlockNumber b = s->relative_block_numbers[i];
261 : :
262 [ # # ]: 0 : if (b < latest_source->truncation_block_length &&
263 [ # # ]: 0 : sourcemap[b] == NULL)
264 : : {
265 : 0 : sourcemap[b] = s;
266 : 0 : offsetmap[b] = s->header_length + (i * BLCKSZ);
267 : :
268 : : /*
269 : : * A full copy of a file from an earlier backup is only
270 : : * possible if no blocks are needed from any later incremental
271 : : * file.
272 : : */
273 : 0 : full_copy_possible = false;
274 : : }
275 : : }
276 : : }
277 : :
278 : : /*
279 : : * If a checksum of the required type already exists in the
280 : : * backup_manifest for the relevant input directory, we can save some work
281 : : * by reusing that checksum instead of computing a new one.
282 : : */
116 rhaas@postgresql.org 283 [ + + + - :GNC 4651 : if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
+ - ]
284 : : checksum_type != CHECKSUM_TYPE_NONE)
285 : : {
286 : : manifest_file *mfile;
287 : :
288 : 4628 : mfile = manifest_files_lookup(manifests[copy_source_index]->files,
289 : : manifest_path);
290 [ - + ]: 4628 : if (mfile == NULL)
291 : : {
116 rhaas@postgresql.org 292 :UNC 0 : char *path = psprintf("%s/backup_manifest",
293 : 0 : prior_backup_dirs[copy_source_index]);
294 : :
295 : : /*
296 : : * The directory is out of sync with the backup_manifest, so emit
297 : : * a warning.
298 : : */
299 : : /*- translator: the first %s is a backup manifest file, the second is a file absent therein */
300 : 0 : pg_log_warning("\"%s\" contains no entry for \"%s\"",
301 : : path,
302 : : manifest_path);
303 : 0 : pfree(path);
304 : : }
116 rhaas@postgresql.org 305 [ + - ]:GNC 4628 : else if (mfile->checksum_type == checksum_type)
306 : : {
307 : 4628 : *checksum_length = mfile->checksum_length;
308 : 4628 : *checksum_payload = pg_malloc(*checksum_length);
309 : 4628 : memcpy(*checksum_payload, mfile->checksum_payload,
310 : 4628 : *checksum_length);
311 : 4628 : checksum_type = CHECKSUM_TYPE_NONE;
312 : : }
313 : : }
314 : :
315 : : /* Prepare for checksum calculation, if required. */
316 : 4651 : pg_checksum_init(&checksum_ctx, checksum_type);
317 : :
318 : : /*
319 : : * If the full file can be created by copying a file from an older backup
320 : : * in the chain without needing to overwrite any blocks or truncate the
321 : : * result, then forget about performing reconstruction and just copy that
322 : : * file in its entirety.
323 : : *
324 : : * Otherwise, reconstruct.
325 : : */
326 [ + + ]: 4651 : if (copy_source != NULL)
327 : 4628 : copy_file(copy_source->filename, output_filename,
328 : : &checksum_ctx, copy_method, dry_run);
329 : : else
330 : : {
331 : 23 : write_reconstructed_file(input_filename, output_filename,
332 : : block_length, sourcemap, offsetmap,
333 : : &checksum_ctx, copy_method,
334 : : debug, dry_run);
335 : 23 : debug_reconstruction(n_prior_backups + 1, source, dry_run);
336 : : }
337 : :
338 : : /* Save results of checksum calculation. */
339 [ + + ]: 4651 : if (checksum_type != CHECKSUM_TYPE_NONE)
340 : : {
341 : 23 : *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
342 : 23 : *checksum_length = pg_checksum_final(&checksum_ctx,
343 : : *checksum_payload);
344 : : }
345 : :
346 : : /*
347 : : * Close files and release memory.
348 : : */
349 [ + + ]: 15282 : for (i = 0; i <= n_prior_backups; ++i)
350 : : {
351 : 10631 : rfile *s = source[i];
352 : :
353 [ - + ]: 10631 : if (s == NULL)
116 rhaas@postgresql.org 354 :UNC 0 : continue;
116 rhaas@postgresql.org 355 [ - + ]:GNC 10631 : if (close(s->fd) != 0)
116 rhaas@postgresql.org 356 :UNC 0 : pg_fatal("could not close \"%s\": %m", s->filename);
116 rhaas@postgresql.org 357 [ + + ]:GNC 10631 : if (s->relative_block_numbers != NULL)
358 : 23 : pfree(s->relative_block_numbers);
359 : 10631 : pg_free(s->filename);
360 : : }
361 : 4651 : pfree(sourcemap);
362 : 4651 : pfree(offsetmap);
363 : 4651 : pfree(source);
364 : 4651 : }
365 : :
366 : : /*
367 : : * Perform post-reconstruction logging and sanity checks.
368 : : */
369 : : static void
370 : 23 : debug_reconstruction(int n_source, rfile **sources, bool dry_run)
371 : : {
372 : : unsigned i;
373 : :
374 [ + + ]: 72 : for (i = 0; i < n_source; ++i)
375 : : {
376 : 49 : rfile *s = sources[i];
377 : :
378 : : /* Ignore source if not used. */
379 [ - + ]: 49 : if (s == NULL)
116 rhaas@postgresql.org 380 :UNC 0 : continue;
381 : :
382 : : /* If no data is needed from this file, we can ignore it. */
116 rhaas@postgresql.org 383 [ + + ]:GNC 49 : if (s->num_blocks_read == 0)
384 : 3 : continue;
385 : :
386 : : /* Debug logging. */
387 [ - + ]: 46 : if (dry_run)
116 rhaas@postgresql.org 388 [ # # ]:UNC 0 : pg_log_debug("would have read %u blocks from \"%s\"",
389 : : s->num_blocks_read, s->filename);
390 : : else
116 rhaas@postgresql.org 391 [ + - ]:GNC 46 : pg_log_debug("read %u blocks from \"%s\"",
392 : : s->num_blocks_read, s->filename);
393 : :
394 : : /*
395 : : * In dry-run mode, we don't actually try to read data from the file,
396 : : * but we do try to verify that the file is long enough that we could
397 : : * have read the data if we'd tried.
398 : : *
399 : : * If this fails, then it means that a non-dry-run attempt would fail,
400 : : * complaining of not being able to read the required bytes from the
401 : : * file.
402 : : */
403 [ - + ]: 46 : if (dry_run)
404 : : {
405 : : struct stat sb;
406 : :
116 rhaas@postgresql.org 407 [ # # ]:UNC 0 : if (fstat(s->fd, &sb) < 0)
408 : 0 : pg_fatal("could not stat \"%s\": %m", s->filename);
409 [ # # ]: 0 : if (sb.st_size < s->highest_offset_read)
410 : 0 : pg_fatal("file \"%s\" is too short: expected %llu, found %llu",
411 : : s->filename,
412 : : (unsigned long long) s->highest_offset_read,
413 : : (unsigned long long) sb.st_size);
414 : : }
415 : : }
116 rhaas@postgresql.org 416 :GNC 23 : }
417 : :
418 : : /*
419 : : * When we perform reconstruction using an incremental file, the output file
420 : : * should be at least as long as the truncation_block_length. Any blocks
421 : : * present in the incremental file increase the output length as far as is
422 : : * necessary to include those blocks.
423 : : */
424 : : static unsigned
425 : 4651 : find_reconstructed_block_length(rfile *s)
426 : : {
427 : 4651 : unsigned block_length = s->truncation_block_length;
428 : : unsigned i;
429 : :
430 [ + + ]: 4686 : for (i = 0; i < s->num_blocks; ++i)
431 [ - + ]: 35 : if (s->relative_block_numbers[i] >= block_length)
116 rhaas@postgresql.org 432 :UNC 0 : block_length = s->relative_block_numbers[i] + 1;
433 : :
116 rhaas@postgresql.org 434 :GNC 4651 : return block_length;
435 : : }
436 : :
437 : : /*
438 : : * Initialize an incremental rfile, reading the header so that we know which
439 : : * blocks it contains.
440 : : */
441 : : static rfile *
442 : 5980 : make_incremental_rfile(char *filename)
443 : : {
444 : : rfile *rf;
445 : : unsigned magic;
446 : :
447 : 5980 : rf = make_rfile(filename, false);
448 : :
449 : : /* Read and validate magic number. */
450 : 5980 : read_bytes(rf, &magic, sizeof(magic));
451 [ - + ]: 5980 : if (magic != INCREMENTAL_MAGIC)
116 rhaas@postgresql.org 452 :UNC 0 : pg_fatal("file \"%s\" has bad incremental magic number (0x%x not 0x%x)",
453 : : filename, magic, INCREMENTAL_MAGIC);
454 : :
455 : : /* Read block count. */
116 rhaas@postgresql.org 456 :GNC 5980 : read_bytes(rf, &rf->num_blocks, sizeof(rf->num_blocks));
457 [ - + ]: 5980 : if (rf->num_blocks > RELSEG_SIZE)
116 rhaas@postgresql.org 458 :UNC 0 : pg_fatal("file \"%s\" has block count %u in excess of segment size %u",
459 : : filename, rf->num_blocks, RELSEG_SIZE);
460 : :
461 : : /* Read truncation block length. */
116 rhaas@postgresql.org 462 :GNC 5980 : read_bytes(rf, &rf->truncation_block_length,
463 : : sizeof(rf->truncation_block_length));
464 [ - + ]: 5980 : if (rf->truncation_block_length > RELSEG_SIZE)
116 rhaas@postgresql.org 465 :UNC 0 : pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
466 : : filename, rf->truncation_block_length, RELSEG_SIZE);
467 : :
468 : : /* Read block numbers if there are any. */
116 rhaas@postgresql.org 469 [ + + ]:GNC 5980 : if (rf->num_blocks > 0)
470 : : {
471 : 23 : rf->relative_block_numbers =
472 : 23 : pg_malloc0(sizeof(BlockNumber) * rf->num_blocks);
473 : 23 : read_bytes(rf, rf->relative_block_numbers,
474 : 23 : sizeof(BlockNumber) * rf->num_blocks);
475 : : }
476 : :
477 : : /* Remember length of header. */
478 : 5980 : rf->header_length = sizeof(magic) + sizeof(rf->num_blocks) +
479 : 5980 : sizeof(rf->truncation_block_length) +
480 : 5980 : sizeof(BlockNumber) * rf->num_blocks;
481 : :
482 : : /*
483 : : * Round header length to a multiple of BLCKSZ, so that blocks contents
484 : : * are properly aligned. Only do this when the file actually has data for
485 : : * some blocks.
486 : : */
9 tomas.vondra@postgre 487 [ + + + - ]: 5980 : if ((rf->num_blocks > 0) && ((rf->header_length % BLCKSZ) != 0))
488 : 23 : rf->header_length += (BLCKSZ - (rf->header_length % BLCKSZ));
489 : :
116 rhaas@postgresql.org 490 : 5980 : return rf;
491 : : }
492 : :
493 : : /*
494 : : * Allocate and perform basic initialization of an rfile.
495 : : */
496 : : static rfile *
497 : 11960 : make_rfile(char *filename, bool missing_ok)
498 : : {
499 : : rfile *rf;
500 : :
501 : 11960 : rf = pg_malloc0(sizeof(rfile));
502 : 11960 : rf->filename = pstrdup(filename);
503 [ + + ]: 11960 : if ((rf->fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0)
504 : : {
505 [ + - + - ]: 1329 : if (missing_ok && errno == ENOENT)
506 : : {
507 : 1329 : pg_free(rf);
508 : 1329 : return NULL;
509 : : }
116 rhaas@postgresql.org 510 :UNC 0 : pg_fatal("could not open file \"%s\": %m", filename);
511 : : }
512 : :
116 rhaas@postgresql.org 513 :GNC 10631 : return rf;
514 : : }
515 : :
516 : : /*
517 : : * Read the indicated number of bytes from an rfile into the buffer.
518 : : */
519 : : static void
520 : 17963 : read_bytes(rfile *rf, void *buffer, unsigned length)
521 : : {
109 tgl@sss.pgh.pa.us 522 : 17963 : int rb = read(rf->fd, buffer, length);
523 : :
116 rhaas@postgresql.org 524 [ - + ]: 17963 : if (rb != length)
525 : : {
116 rhaas@postgresql.org 526 [ # # ]:UNC 0 : if (rb < 0)
527 : 0 : pg_fatal("could not read file \"%s\": %m", rf->filename);
528 : : else
32 peter@eisentraut.org 529 : 0 : pg_fatal("could not read file \"%s\": read only %d of %u bytes",
530 : : rf->filename, rb, length);
531 : : }
116 rhaas@postgresql.org 532 :GNC 17963 : }
533 : :
534 : : /*
535 : : * Write out a reconstructed file.
536 : : */
537 : : static void
538 : 23 : write_reconstructed_file(char *input_filename,
539 : : char *output_filename,
540 : : unsigned block_length,
541 : : rfile **sourcemap,
542 : : off_t *offsetmap,
543 : : pg_checksum_context *checksum_ctx,
544 : : CopyMethod copy_method,
545 : : bool debug,
546 : : bool dry_run)
547 : : {
548 : 23 : int wfd = -1;
549 : : unsigned i;
550 : 23 : unsigned zero_blocks = 0;
551 : :
552 : : /* Debugging output. */
553 [ + - ]: 23 : if (debug)
554 : : {
555 : : StringInfoData debug_buf;
556 : 23 : unsigned start_of_range = 0;
557 : 23 : unsigned current_block = 0;
558 : :
559 : : /* Basic information about the output file to be produced. */
560 [ - + ]: 23 : if (dry_run)
116 rhaas@postgresql.org 561 [ # # ]:UNC 0 : pg_log_debug("would reconstruct \"%s\" (%u blocks, checksum %s)",
562 : : output_filename, block_length,
563 : : pg_checksum_type_name(checksum_ctx->type));
564 : : else
116 rhaas@postgresql.org 565 [ + - ]:GNC 23 : pg_log_debug("reconstructing \"%s\" (%u blocks, checksum %s)",
566 : : output_filename, block_length,
567 : : pg_checksum_type_name(checksum_ctx->type));
568 : :
569 : : /* Print out the plan for reconstructing this file. */
570 : 23 : initStringInfo(&debug_buf);
571 [ + + ]: 277 : while (current_block < block_length)
572 : : {
573 : 254 : rfile *s = sourcemap[current_block];
574 : :
575 : : /* Extend range, if possible. */
576 [ + + ]: 254 : if (current_block + 1 < block_length &&
577 [ + + ]: 231 : s == sourcemap[current_block + 1])
578 : : {
579 : 199 : ++current_block;
580 : 199 : continue;
581 : : }
582 : :
583 : : /* Add details about this range. */
584 [ - + ]: 55 : if (s == NULL)
585 : : {
116 rhaas@postgresql.org 586 [ # # ]:UNC 0 : if (current_block == start_of_range)
587 : 0 : appendStringInfo(&debug_buf, " %u:zero", current_block);
588 : : else
589 : 0 : appendStringInfo(&debug_buf, " %u-%u:zero",
590 : : start_of_range, current_block);
591 : : }
592 : : else
593 : : {
116 rhaas@postgresql.org 594 [ + + ]:GNC 55 : if (current_block == start_of_range)
595 : 36 : appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT,
596 : : current_block, s->filename,
597 : 36 : (uint64) offsetmap[current_block]);
598 : : else
599 : 19 : appendStringInfo(&debug_buf, " %u-%u:%s@" UINT64_FORMAT,
600 : : start_of_range, current_block,
601 : : s->filename,
602 : 19 : (uint64) offsetmap[current_block]);
603 : : }
604 : :
605 : : /* Begin new range. */
606 : 55 : start_of_range = ++current_block;
607 : :
608 : : /* If the output is very long or we are done, dump it now. */
609 [ + + - + ]: 55 : if (current_block == block_length || debug_buf.len > 1024)
610 : : {
611 [ + - ]: 23 : pg_log_debug("reconstruction plan:%s", debug_buf.data);
612 : 23 : resetStringInfo(&debug_buf);
613 : : }
614 : : }
615 : :
616 : : /* Free memory. */
617 : 23 : pfree(debug_buf.data);
618 : : }
619 : :
620 : : /* Open the output file, except in dry_run mode. */
621 [ + - - + ]: 46 : if (!dry_run &&
622 : 23 : (wfd = open(output_filename,
623 : : O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
624 : : pg_file_create_mode)) < 0)
116 rhaas@postgresql.org 625 :UNC 0 : pg_fatal("could not open file \"%s\": %m", output_filename);
626 : :
627 : : /* Read and write the blocks as required. */
116 rhaas@postgresql.org 628 [ + + ]:GNC 277 : for (i = 0; i < block_length; ++i)
629 : : {
630 : : uint8 buffer[BLCKSZ];
631 : 254 : rfile *s = sourcemap[i];
632 : :
633 : : /* Update accounting information. */
634 [ - + ]: 254 : if (s == NULL)
116 rhaas@postgresql.org 635 :UNC 0 : ++zero_blocks;
636 : : else
637 : : {
116 rhaas@postgresql.org 638 :GNC 254 : s->num_blocks_read++;
639 : 254 : s->highest_offset_read = Max(s->highest_offset_read,
640 : : offsetmap[i] + BLCKSZ);
641 : : }
642 : :
643 : : /* Skip the rest of this in dry-run mode. */
644 [ - + ]: 254 : if (dry_run)
116 rhaas@postgresql.org 645 :UNC 0 : continue;
646 : :
647 : : /* Read or zero-fill the block as appropriate. */
116 rhaas@postgresql.org 648 [ - + ]:GNC 254 : if (s == NULL)
649 : : {
650 : : /*
651 : : * New block not mentioned in the WAL summary. Should have been an
652 : : * uninitialized block, so just zero-fill it.
653 : : */
116 rhaas@postgresql.org 654 :UNC 0 : memset(buffer, 0, BLCKSZ);
655 : :
656 : : /* Write out the block, update the checksum if needed. */
9 tomas.vondra@postgre 657 : 0 : write_block(wfd, output_filename, buffer, checksum_ctx);
658 : :
659 : : /* Nothing else to do for zero-filled blocks. */
660 : 0 : continue;
661 : : }
662 : :
663 : : /* Copy the block using the appropriate copy method. */
9 tomas.vondra@postgre 664 [ + - ]:GNC 254 : if (copy_method != COPY_METHOD_COPY_FILE_RANGE)
665 : : {
666 : : /*
667 : : * Read the block from the correct source file, and then write it
668 : : * out, possibly with a checksum update.
669 : : */
670 : 254 : read_block(s, offsetmap[i], buffer);
671 : 254 : write_block(wfd, output_filename, buffer, checksum_ctx);
672 : : }
673 : : else /* use copy_file_range */
674 : : {
675 : : #if defined(HAVE_COPY_FILE_RANGE)
676 : : /* copy_file_range modifies the offset, so use a local copy */
9 tomas.vondra@postgre 677 :UNC 0 : off_t off = offsetmap[i];
678 : 0 : size_t nwritten = 0;
679 : :
680 : : /*
681 : : * Retry until we've written all the bytes (the offset is updated
682 : : * by copy_file_range, and so is the wfd file offset).
683 : : */
684 : : do
685 : : {
686 : : int wb;
687 : :
688 : 0 : wb = copy_file_range(s->fd, &off, wfd, NULL, BLCKSZ - nwritten, 0);
689 : :
690 [ # # ]: 0 : if (wb < 0)
691 : 0 : pg_fatal("error while copying file range from \"%s\" to \"%s\": %m",
692 : : input_filename, output_filename);
693 : :
694 : 0 : nwritten += wb;
695 : :
696 [ # # ]: 0 : } while (BLCKSZ > nwritten);
697 : :
698 : : /*
699 : : * When checksum calculation not needed, we're done, otherwise
700 : : * read the block and pass it to the checksum calculation.
701 : : */
702 [ # # ]: 0 : if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
703 : 0 : continue;
704 : :
705 : 0 : read_block(s, offsetmap[i], buffer);
706 : :
707 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
708 : 0 : pg_fatal("could not update checksum of file \"%s\"",
709 : : output_filename);
710 : : #else
711 : : pg_fatal("copy_file_range not supported on this platform");
712 : : #endif
713 : : }
714 : : }
715 : :
716 : : /* Debugging output. */
116 rhaas@postgresql.org 717 [ - + ]:GNC 23 : if (zero_blocks > 0)
718 : : {
116 rhaas@postgresql.org 719 [ # # ]:UNC 0 : if (dry_run)
720 [ # # ]: 0 : pg_log_debug("would have zero-filled %u blocks", zero_blocks);
721 : : else
722 [ # # ]: 0 : pg_log_debug("zero-filled %u blocks", zero_blocks);
723 : : }
724 : :
725 : : /* Close the output file. */
116 rhaas@postgresql.org 726 [ + - - + ]:GNC 23 : if (wfd >= 0 && close(wfd) != 0)
116 rhaas@postgresql.org 727 :UNC 0 : pg_fatal("could not close \"%s\": %m", output_filename);
116 rhaas@postgresql.org 728 :GNC 23 : }
729 : :
730 : : /*
731 : : * Write the block into the file (using the file descriptor), and
732 : : * if needed update the checksum calculation.
733 : : *
734 : : * The buffer is expected to contain BLCKSZ bytes. The filename is
735 : : * provided only for the error message.
736 : : */
737 : : static void
9 tomas.vondra@postgre 738 : 254 : write_block(int fd, char *output_filename,
739 : : uint8 *buffer, pg_checksum_context *checksum_ctx)
740 : : {
741 : : int wb;
742 : :
743 [ - + ]: 254 : if ((wb = write(fd, buffer, BLCKSZ)) != BLCKSZ)
744 : : {
9 tomas.vondra@postgre 745 [ # # ]:UNC 0 : if (wb < 0)
746 : 0 : pg_fatal("could not write file \"%s\": %m", output_filename);
747 : : else
748 : 0 : pg_fatal("could not write file \"%s\": wrote only %d of %d bytes",
749 : : output_filename, wb, BLCKSZ);
750 : : }
751 : :
752 : : /* Update the checksum computation. */
9 tomas.vondra@postgre 753 [ - + ]:GNC 254 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
9 tomas.vondra@postgre 754 :UNC 0 : pg_fatal("could not update checksum of file \"%s\"",
755 : : output_filename);
9 tomas.vondra@postgre 756 :GNC 254 : }
757 : :
758 : : /*
759 : : * Read a block of data (BLCKSZ bytes) into the the buffer.
760 : : */
761 : : static void
762 : 254 : read_block(rfile *s, off_t off, uint8 *buffer)
763 : : {
764 : : int rb;
765 : :
766 : : /* Read the block from the correct source, except if dry-run. */
767 : 254 : rb = pg_pread(s->fd, buffer, BLCKSZ, off);
768 [ - + ]: 254 : if (rb != BLCKSZ)
769 : : {
9 tomas.vondra@postgre 770 [ # # ]:UNC 0 : if (rb < 0)
771 : 0 : pg_fatal("could not read file \"%s\": %m", s->filename);
772 : : else
773 : 0 : pg_fatal("could not read file \"%s\": read only %d of %d bytes at offset %llu",
774 : : s->filename, rb, BLCKSZ,
775 : : (unsigned long long) off);
776 : : }
9 tomas.vondra@postgre 777 :GNC 254 : }
|
