Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * file_ops.c
4 : * Helper functions for operating on files.
5 : *
6 : * Most of the functions in this file are helper functions for writing to
7 : * the target data directory. The functions check the --dry-run flag, and
8 : * do nothing if it's enabled. You should avoid accessing the target files
9 : * directly but if you do, make sure you honor the --dry-run mode!
10 : *
11 : * Portions Copyright (c) 2013-2023, PostgreSQL Global Development Group
12 : *
13 : *-------------------------------------------------------------------------
14 : */
15 : #include "postgres_fe.h"
16 :
17 : #include <sys/stat.h>
18 : #include <dirent.h>
19 : #include <fcntl.h>
20 : #include <unistd.h>
21 :
22 : #include "common/file_perm.h"
23 : #include "common/file_utils.h"
24 : #include "file_ops.h"
25 : #include "filemap.h"
26 : #include "pg_rewind.h"
27 :
28 : /*
29 : * Currently open target file.
30 : */
31 : static int dstfd = -1;
32 : static char dstpath[MAXPGPATH] = "";
33 :
34 : static void create_target_dir(const char *path);
35 : static void remove_target_dir(const char *path);
36 : static void create_target_symlink(const char *path, const char *link);
37 : static void remove_target_symlink(const char *path);
38 :
39 : static void recurse_dir(const char *datadir, const char *parentpath,
40 : process_file_callback_t callback);
41 :
42 : /*
43 : * Open a target file for writing. If 'trunc' is true and the file already
44 : * exists, it will be truncated.
45 : */
46 : void
2939 heikki.linnakangas 47 CBC 7441 : open_target_file(const char *path, bool trunc)
48 : {
49 : int mode;
50 :
51 7441 : if (dry_run)
52 268 : return;
53 :
54 7173 : if (dstfd != -1 && !trunc &&
55 3237 : strcmp(path, &dstpath[strlen(datadir_target) + 1]) == 0)
56 896 : return; /* already open */
57 :
58 6277 : close_target_file();
59 :
60 6277 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
61 :
62 6277 : mode = O_WRONLY | O_CREAT | PG_BINARY;
63 6277 : if (trunc)
64 3936 : mode |= O_TRUNC;
1828 sfrost 65 6277 : dstfd = open(dstpath, mode, pg_file_create_mode);
2939 heikki.linnakangas 66 6277 : if (dstfd < 0)
1469 peter 67 UBC 0 : pg_fatal("could not open target file \"%s\": %m",
68 : dstpath);
69 : }
70 :
71 : /*
72 : * Close target file, if it's open.
73 : */
74 : void
2939 heikki.linnakangas 75 CBC 6301 : close_target_file(void)
76 : {
77 6301 : if (dstfd == -1)
78 25 : return;
79 :
80 6276 : if (close(dstfd) != 0)
1469 peter 81 UBC 0 : pg_fatal("could not close target file \"%s\": %m",
82 : dstpath);
83 :
2939 heikki.linnakangas 84 CBC 6276 : dstfd = -1;
85 : }
86 :
87 : void
88 49114 : write_target_range(char *buf, off_t begin, size_t size)
89 : {
90 : size_t writeleft;
91 : char *p;
92 :
93 : /* update progress report */
94 49114 : fetch_done += size;
95 49114 : progress_report(false);
96 :
97 49114 : if (dry_run)
98 6728 : return;
99 :
100 42386 : if (lseek(dstfd, begin, SEEK_SET) == -1)
1469 peter 101 UBC 0 : pg_fatal("could not seek in target file \"%s\": %m",
102 : dstpath);
103 :
2939 heikki.linnakangas 104 CBC 42386 : writeleft = size;
105 42386 : p = buf;
106 84715 : while (writeleft > 0)
107 : {
108 : ssize_t writelen;
109 :
2859 fujii 110 42329 : errno = 0;
2939 heikki.linnakangas 111 42329 : writelen = write(dstfd, p, writeleft);
112 42329 : if (writelen < 0)
113 : {
114 : /* if write didn't set errno, assume problem is no disk space */
2859 fujii 115 UBC 0 : if (errno == 0)
116 0 : errno = ENOSPC;
1469 peter 117 0 : pg_fatal("could not write file \"%s\": %m",
118 : dstpath);
119 : }
120 :
2939 heikki.linnakangas 121 CBC 42329 : p += writelen;
122 42329 : writeleft -= writelen;
123 : }
124 :
125 : /* keep the file open, in case we need to copy more blocks in it */
126 : }
127 :
128 :
129 : void
130 700 : remove_target(file_entry_t *entry)
131 : {
132 700 : Assert(entry->action == FILE_ACTION_REMOVE);
886 133 700 : Assert(entry->target_exists);
134 :
135 700 : switch (entry->target_type)
136 : {
2939 137 16 : case FILE_TYPE_DIRECTORY:
138 16 : remove_target_dir(entry->path);
139 16 : break;
140 :
141 684 : case FILE_TYPE_REGULAR:
1837 fujii 142 684 : remove_target_file(entry->path, false);
2939 heikki.linnakangas 143 684 : break;
144 :
2939 heikki.linnakangas 145 UBC 0 : case FILE_TYPE_SYMLINK:
146 0 : remove_target_symlink(entry->path);
147 0 : break;
148 :
886 149 0 : case FILE_TYPE_UNDEFINED:
150 0 : pg_fatal("undefined file type for \"%s\"", entry->path);
151 : break;
152 : }
2939 heikki.linnakangas 153 CBC 700 : }
154 :
155 : void
156 8 : create_target(file_entry_t *entry)
157 : {
158 8 : Assert(entry->action == FILE_ACTION_CREATE);
886 159 8 : Assert(!entry->target_exists);
160 :
161 8 : switch (entry->source_type)
162 : {
2939 163 8 : case FILE_TYPE_DIRECTORY:
164 8 : create_target_dir(entry->path);
165 8 : break;
166 :
2939 heikki.linnakangas 167 UBC 0 : case FILE_TYPE_SYMLINK:
886 168 0 : create_target_symlink(entry->path, entry->source_link_target);
2939 169 0 : break;
170 :
171 0 : case FILE_TYPE_REGULAR:
172 : /* can't happen. Regular files are created with open_target_file. */
1469 peter 173 0 : pg_fatal("invalid action (CREATE) for regular file");
174 : break;
175 :
886 heikki.linnakangas 176 0 : case FILE_TYPE_UNDEFINED:
177 0 : pg_fatal("undefined file type for \"%s\"", entry->path);
178 : break;
179 : }
2939 heikki.linnakangas 180 CBC 8 : }
181 :
182 : /*
183 : * Remove a file from target data directory. If missing_ok is true, it
184 : * is fine for the target file to not exist.
185 : */
186 : void
1837 fujii 187 684 : remove_target_file(const char *path, bool missing_ok)
188 : {
189 : char dstpath[MAXPGPATH];
190 :
2939 heikki.linnakangas 191 684 : if (dry_run)
192 7 : return;
193 :
194 677 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
195 677 : if (unlink(dstpath) != 0)
196 : {
1837 fujii 197 UBC 0 : if (errno == ENOENT && missing_ok)
198 0 : return;
199 :
1469 peter 200 0 : pg_fatal("could not remove file \"%s\": %m",
201 : dstpath);
202 : }
203 : }
204 :
205 : void
2939 heikki.linnakangas 206 CBC 4 : truncate_target_file(const char *path, off_t newsize)
207 : {
208 : char dstpath[MAXPGPATH];
209 : int fd;
210 :
211 4 : if (dry_run)
212 1 : return;
213 :
214 3 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
215 :
1828 sfrost 216 3 : fd = open(dstpath, O_WRONLY, pg_file_create_mode);
2939 heikki.linnakangas 217 3 : if (fd < 0)
1469 peter 218 UBC 0 : pg_fatal("could not open file \"%s\" for truncation: %m",
219 : dstpath);
220 :
2939 heikki.linnakangas 221 CBC 3 : if (ftruncate(fd, newsize) != 0)
1469 peter 222 UBC 0 : pg_fatal("could not truncate file \"%s\" to %u: %m",
223 : dstpath, (unsigned int) newsize);
224 :
2939 heikki.linnakangas 225 CBC 3 : close(fd);
226 : }
227 :
228 : static void
229 8 : create_target_dir(const char *path)
230 : {
231 : char dstpath[MAXPGPATH];
232 :
233 8 : if (dry_run)
2939 heikki.linnakangas 234 UBC 0 : return;
235 :
2939 heikki.linnakangas 236 CBC 8 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
1828 sfrost 237 8 : if (mkdir(dstpath, pg_dir_create_mode) != 0)
1469 peter 238 UBC 0 : pg_fatal("could not create directory \"%s\": %m",
239 : dstpath);
240 : }
241 :
242 : static void
2939 heikki.linnakangas 243 CBC 16 : remove_target_dir(const char *path)
244 : {
245 : char dstpath[MAXPGPATH];
246 :
247 16 : if (dry_run)
248 1 : return;
249 :
250 15 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
251 15 : if (rmdir(dstpath) != 0)
1469 peter 252 UBC 0 : pg_fatal("could not remove directory \"%s\": %m",
253 : dstpath);
254 : }
255 :
256 : static void
2939 heikki.linnakangas 257 0 : create_target_symlink(const char *path, const char *link)
258 : {
259 : char dstpath[MAXPGPATH];
260 :
261 0 : if (dry_run)
262 0 : return;
263 :
264 0 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
265 0 : if (symlink(link, dstpath) != 0)
1469 peter 266 0 : pg_fatal("could not create symbolic link at \"%s\": %m",
267 : dstpath);
268 : }
269 :
270 : static void
2939 heikki.linnakangas 271 0 : remove_target_symlink(const char *path)
272 : {
273 : char dstpath[MAXPGPATH];
274 :
275 0 : if (dry_run)
276 0 : return;
277 :
278 0 : snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
279 0 : if (unlink(dstpath) != 0)
1469 peter 280 0 : pg_fatal("could not remove symbolic link \"%s\": %m",
281 : dstpath);
282 : }
283 :
284 : /*
285 : * Sync target data directory to ensure that modifications are safely on disk.
286 : *
287 : * We do this once, for the whole data directory, for performance reasons. At
288 : * the end of pg_rewind's run, the kernel is likely to already have flushed
289 : * most dirty buffers to disk. Additionally fsync_pgdata uses a two-pass
290 : * approach (only initiating writeback in the first pass), which often reduces
291 : * the overall amount of IO noticeably.
292 : */
293 : void
886 heikki.linnakangas 294 CBC 12 : sync_target_dir(void)
295 : {
296 12 : if (!do_sync || dry_run)
297 11 : return;
298 :
299 1 : fsync_pgdata(datadir_target, PG_VERSION_NUM);
300 : }
301 :
302 :
303 : /*
304 : * Read a file into memory. The file to be read is <datadir>/<path>.
305 : * The file contents are returned in a malloc'd buffer, and *filesize
306 : * is set to the length of the file.
307 : *
308 : * The returned buffer is always zero-terminated; the size of the returned
309 : * buffer is actually *filesize + 1. That's handy when reading a text file.
310 : * This function can be used to read binary files as well, you can just
311 : * ignore the zero-terminator in that case.
312 : */
313 : char *
2939 314 51 : slurpFile(const char *datadir, const char *path, size_t *filesize)
315 : {
316 : int fd;
317 : char *buffer;
318 : struct stat statbuf;
319 : char fullpath[MAXPGPATH];
320 : int len;
321 : int r;
322 :
323 51 : snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
324 :
325 51 : if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1)
1469 peter 326 UBC 0 : pg_fatal("could not open file \"%s\" for reading: %m",
327 : fullpath);
328 :
2939 heikki.linnakangas 329 CBC 51 : if (fstat(fd, &statbuf) < 0)
1469 peter 330 UBC 0 : pg_fatal("could not open file \"%s\" for reading: %m",
331 : fullpath);
332 :
2939 heikki.linnakangas 333 CBC 51 : len = statbuf.st_size;
334 :
335 51 : buffer = pg_malloc(len + 1);
336 :
1726 michael 337 51 : r = read(fd, buffer, len);
338 51 : if (r != len)
339 : {
1726 michael 340 UBC 0 : if (r < 0)
1469 peter 341 0 : pg_fatal("could not read file \"%s\": %m",
342 : fullpath);
343 : else
344 0 : pg_fatal("could not read file \"%s\": read %d of %zu",
345 : fullpath, r, (Size) len);
346 : }
2939 heikki.linnakangas 347 CBC 51 : close(fd);
348 :
349 : /* Zero-terminate the buffer. */
350 51 : buffer[len] = '\0';
351 :
352 51 : if (filesize)
353 42 : *filesize = len;
354 51 : return buffer;
355 : }
356 :
357 : /*
358 : * Traverse through all files in a data directory, calling 'callback'
359 : * for each file.
360 : */
361 : void
886 362 20 : traverse_datadir(const char *datadir, process_file_callback_t callback)
363 : {
364 20 : recurse_dir(datadir, NULL, callback);
365 20 : }
366 :
367 : /*
368 : * recursive part of traverse_datadir
369 : *
370 : * parentpath is the current subdirectory's path relative to datadir,
371 : * or NULL at the top level.
372 : */
373 : static void
374 552 : recurse_dir(const char *datadir, const char *parentpath,
375 : process_file_callback_t callback)
376 : {
377 : DIR *xldir;
378 : struct dirent *xlde;
379 : char fullparentpath[MAXPGPATH];
380 :
381 552 : if (parentpath)
382 532 : snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
383 : else
384 20 : snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
385 :
386 552 : xldir = opendir(fullparentpath);
387 552 : if (xldir == NULL)
886 heikki.linnakangas 388 UBC 0 : pg_fatal("could not open directory \"%s\": %m",
389 : fullparentpath);
390 :
886 heikki.linnakangas 391 CBC 24500 : while (errno = 0, (xlde = readdir(xldir)) != NULL)
392 : {
393 : struct stat fst;
394 : char fullpath[MAXPGPATH * 2];
395 : char path[MAXPGPATH * 2];
396 :
397 23948 : if (strcmp(xlde->d_name, ".") == 0 ||
398 23396 : strcmp(xlde->d_name, "..") == 0)
399 1104 : continue;
400 :
401 22844 : snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
402 :
403 22844 : if (lstat(fullpath, &fst) < 0)
404 : {
886 heikki.linnakangas 405 UBC 0 : if (errno == ENOENT)
406 : {
407 : /*
408 : * File doesn't exist anymore. This is ok, if the new primary
409 : * is running and the file was just removed. If it was a data
410 : * file, there should be a WAL record of the removal. If it
411 : * was something else, it couldn't have been anyway.
412 : *
413 : * TODO: But complain if we're processing the target dir!
414 : */
415 : }
416 : else
417 0 : pg_fatal("could not stat file \"%s\": %m",
418 : fullpath);
419 : }
420 :
886 heikki.linnakangas 421 CBC 22844 : if (parentpath)
422 22357 : snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
423 : else
424 487 : snprintf(path, sizeof(path), "%s", xlde->d_name);
425 :
426 22844 : if (S_ISREG(fst.st_mode))
427 22312 : callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
428 532 : else if (S_ISDIR(fst.st_mode))
429 : {
430 530 : callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
431 : /* recurse to handle subdirectories */
432 530 : recurse_dir(datadir, path, callback);
433 : }
886 heikki.linnakangas 434 GIC 2 : else if (S_ISLNK(fst.st_mode))
886 heikki.linnakangas 435 ECB : {
886 heikki.linnakangas 436 EUB : char link_target[MAXPGPATH];
437 : int len;
886 heikki.linnakangas 438 ECB :
886 heikki.linnakangas 439 GBC 2 : len = readlink(fullpath, link_target, sizeof(link_target));
886 heikki.linnakangas 440 GIC 2 : if (len < 0)
886 heikki.linnakangas 441 LBC 0 : pg_fatal("could not read symbolic link \"%s\": %m",
442 : fullpath);
886 heikki.linnakangas 443 CBC 2 : if (len >= sizeof(link_target))
886 heikki.linnakangas 444 UIC 0 : pg_fatal("symbolic link \"%s\" target is too long",
445 : fullpath);
886 heikki.linnakangas 446 GIC 2 : link_target[len] = '\0';
447 :
448 2 : callback(path, FILE_TYPE_SYMLINK, 0, link_target);
449 :
886 heikki.linnakangas 450 ECB : /*
451 : * If it's a symlink within pg_tblspc, we need to recurse into it,
452 : * to process all the tablespaces. We also follow a symlink if
453 : * it's for pg_wal. Symlinks elsewhere are ignored.
454 : */
886 heikki.linnakangas 455 GIC 2 : if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
886 heikki.linnakangas 456 CBC 2 : strcmp(path, "pg_wal") == 0)
886 heikki.linnakangas 457 GBC 2 : recurse_dir(datadir, path, callback);
458 : }
886 heikki.linnakangas 459 ECB : }
460 :
886 heikki.linnakangas 461 GIC 552 : if (errno)
886 heikki.linnakangas 462 UIC 0 : pg_fatal("could not read directory \"%s\": %m",
463 : fullparentpath);
464 :
886 heikki.linnakangas 465 GIC 552 : if (closedir(xldir))
886 heikki.linnakangas 466 UIC 0 : pg_fatal("could not close directory \"%s\": %m",
467 : fullparentpath);
886 heikki.linnakangas 468 GIC 552 : }
|
