Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * File-processing utility routines.
4 : *
5 : * Assorted utility functions to work on files.
6 : *
7 : *
8 : * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
9 : * Portions Copyright (c) 1994, Regents of the University of California
10 : *
11 : * src/common/file_utils.c
12 : *
13 : *-------------------------------------------------------------------------
14 : */
15 :
16 : #ifndef FRONTEND
17 : #include "postgres.h"
18 : #else
19 : #include "postgres_fe.h"
20 : #endif
21 :
22 : #include <dirent.h>
23 : #include <fcntl.h>
24 : #include <sys/stat.h>
25 : #include <unistd.h>
26 :
27 : #include "common/file_utils.h"
28 : #ifdef FRONTEND
29 : #include "common/logging.h"
30 : #endif
31 : #include "port/pg_iovec.h"
32 :
33 : #ifdef FRONTEND
34 :
35 : /* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */
36 : #if defined(HAVE_SYNC_FILE_RANGE)
37 : #define PG_FLUSH_DATA_WORKS 1
38 : #elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
39 : #define PG_FLUSH_DATA_WORKS 1
40 : #endif
41 :
42 : /*
43 : * pg_xlog has been renamed to pg_wal in version 10.
44 : */
45 : #define MINIMUM_VERSION_FOR_PG_WAL 100000
46 :
47 : #ifdef PG_FLUSH_DATA_WORKS
48 : static int pre_sync_fname(const char *fname, bool isdir);
49 : #endif
50 : static void walkdir(const char *path,
51 : int (*action) (const char *fname, bool isdir),
52 : bool process_symlinks);
53 :
54 : /*
55 : * Issue fsync recursively on PGDATA and all its contents.
56 : *
57 : * We fsync regular files and directories wherever they are, but we follow
58 : * symlinks only for pg_wal (or pg_xlog) and immediately under pg_tblspc.
59 : * Other symlinks are presumed to point at files we're not responsible for
60 : * fsyncing, and might not have privileges to write at all.
61 : *
62 : * serverVersion indicates the version of the server to be fsync'd.
63 : */
64 : void
2362 rhaas 65 GIC 4 : fsync_pgdata(const char *pg_data,
2362 rhaas 66 ECB : int serverVersion)
67 : {
68 : bool xlog_is_symlink;
69 : char pg_wal[MAXPGPATH];
70 : char pg_tblspc[MAXPGPATH];
71 :
72 : /* handle renaming of pg_xlog to pg_wal in post-10 clusters */
2362 rhaas 73 GIC 4 : snprintf(pg_wal, MAXPGPATH, "%s/%s", pg_data,
2118 tgl 74 ECB : serverVersion < MINIMUM_VERSION_FOR_PG_WAL ? "pg_xlog" : "pg_wal");
2383 peter_e 75 GIC 4 : snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data);
2383 peter_e 76 ECB :
77 : /*
78 : * If pg_wal is a symlink, we'll need to recurse into it separately,
79 : * because the first walkdir below will ignore it.
80 : */
2383 peter_e 81 GIC 4 : xlog_is_symlink = false;
2383 peter_e 82 ECB :
83 : {
84 : struct stat st;
85 :
2362 rhaas 86 CBC 4 : if (lstat(pg_wal, &st) < 0)
1469 peter 87 UBC 0 : pg_log_error("could not stat file \"%s\": %m", pg_wal);
2383 peter_e 88 CBC 4 : else if (S_ISLNK(st.st_mode))
89 1 : xlog_is_symlink = true;
90 : }
91 :
92 : /*
2383 peter_e 93 ECB : * If possible, hint to the kernel that we're soon going to fsync the data
94 : * directory and its contents.
95 : */
96 : #ifdef PG_FLUSH_DATA_WORKS
1469 peter 97 GIC 4 : walkdir(pg_data, pre_sync_fname, false);
2383 peter_e 98 4 : if (xlog_is_symlink)
1469 peter 99 1 : walkdir(pg_wal, pre_sync_fname, false);
100 4 : walkdir(pg_tblspc, pre_sync_fname, true);
101 : #endif
102 :
103 : /*
104 : * Now we do the fsync()s in the same order.
105 : *
106 : * The main call ignores symlinks, so in addition to specially processing
107 : * pg_wal if it's a symlink, pg_tblspc has to be visited separately with
2383 peter_e 108 ECB : * process_symlinks = true. Note that if there are any plain directories
109 : * in pg_tblspc, they'll get fsync'd twice. That's not an expected case
110 : * so we don't worry about optimizing it.
111 : */
1469 peter 112 CBC 4 : walkdir(pg_data, fsync_fname, false);
2383 peter_e 113 GIC 4 : if (xlog_is_symlink)
1469 peter 114 1 : walkdir(pg_wal, fsync_fname, false);
115 4 : walkdir(pg_tblspc, fsync_fname, true);
2383 peter_e 116 4 : }
117 :
118 : /*
119 : * Issue fsync recursively on the given directory and all its contents.
2209 andrew 120 ECB : *
121 : * This is a convenient wrapper on top of walkdir().
122 : */
123 : void
1469 peter 124 GIC 5 : fsync_dir_recurse(const char *dir)
125 : {
126 : /*
2209 andrew 127 ECB : * If possible, hint to the kernel that we're soon going to fsync the data
128 : * directory and its contents.
129 : */
130 : #ifdef PG_FLUSH_DATA_WORKS
1469 peter 131 CBC 5 : walkdir(dir, pre_sync_fname, false);
132 : #endif
133 :
1469 peter 134 GIC 5 : walkdir(dir, fsync_fname, false);
2209 andrew 135 5 : }
136 :
137 : /*
138 : * walkdir: recursively walk a directory, applying the action to each
139 : * regular file and directory (including the named directory itself).
140 : *
141 : * If process_symlinks is true, the action and recursion are also applied
142 : * to regular files and directories that are pointed to by symlinks in the
143 : * given directory; otherwise symlinks are ignored. Symlinks are always
144 : * ignored in subdirectories, ie we intentionally don't pass down the
145 : * process_symlinks flag to recursive calls.
146 : *
147 : * Errors are reported but not considered fatal.
2383 peter_e 148 ECB : *
149 : * See also walkdir in fd.c, which is a backend version of this logic.
150 : */
151 : static void
2383 peter_e 152 GIC 228 : walkdir(const char *path,
153 : int (*action) (const char *fname, bool isdir),
154 : bool process_symlinks)
2383 peter_e 155 ECB : {
156 : DIR *dir;
157 : struct dirent *de;
2383 peter_e 158 EUB :
2383 peter_e 159 GBC 228 : dir = opendir(path);
2383 peter_e 160 GIC 228 : if (dir == NULL)
161 : {
1469 peter 162 LBC 0 : pg_log_error("could not open directory \"%s\": %m", path);
2383 peter_e 163 UIC 0 : return;
164 : }
165 :
2383 peter_e 166 CBC 8960 : while (errno = 0, (de = readdir(dir)) != NULL)
2383 peter_e 167 ECB : {
2189 168 : char subpath[MAXPGPATH * 2];
169 :
2383 peter_e 170 CBC 8732 : if (strcmp(de->d_name, ".") == 0 ||
2383 peter_e 171 GIC 8504 : strcmp(de->d_name, "..") == 0)
2383 peter_e 172 CBC 456 : continue;
173 :
2189 174 8276 : snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
2383 peter_e 175 ECB :
944 tmunro 176 CBC 8276 : switch (get_dirent_type(subpath, de, process_symlinks, PG_LOG_ERROR))
2383 peter_e 177 ECB : {
944 tmunro 178 CBC 8074 : case PGFILETYPE_REG:
179 8074 : (*action) (subpath, false);
180 8074 : break;
944 tmunro 181 GIC 200 : case PGFILETYPE_DIR:
182 200 : walkdir(subpath, action, false);
183 200 : break;
184 2 : default:
185 :
186 : /*
944 tmunro 187 ECB : * Errors are already reported directly by get_dirent_type(),
188 : * and any remaining symlinks and unknown file types are
189 : * ignored.
190 : */
944 tmunro 191 CBC 2 : break;
2383 peter_e 192 EUB : }
193 : }
2383 peter_e 194 ECB :
2383 peter_e 195 GIC 228 : if (errno)
1469 peter 196 UIC 0 : pg_log_error("could not read directory \"%s\": %m", path);
197 :
2383 peter_e 198 GIC 228 : (void) closedir(dir);
199 :
200 : /*
201 : * It's important to fsync the destination directory itself as individual
2383 peter_e 202 ECB : * file fsyncs don't guarantee that the directory entry for the file is
203 : * synced. Recent versions of ext4 have made the window much wider but
204 : * it's been an issue for ext3 and other filesystems in the past.
205 : */
1469 peter 206 GIC 228 : (*action) (path, true);
207 : }
208 :
209 : /*
210 : * Hint to the OS that it should get ready to fsync() this file.
211 : *
212 : * Ignores errors trying to open unreadable files, and reports other errors
213 : * non-fatally.
2383 peter_e 214 ECB : */
215 : #ifdef PG_FLUSH_DATA_WORKS
216 :
217 : static int
1469 peter 218 CBC 4151 : pre_sync_fname(const char *fname, bool isdir)
219 : {
2383 peter_e 220 ECB : int fd;
221 :
1668 michael 222 GBC 4151 : fd = open(fname, O_RDONLY | PG_BINARY, 0);
2383 peter_e 223 EUB :
2383 peter_e 224 GBC 4151 : if (fd < 0)
2383 peter_e 225 EUB : {
2383 peter_e 226 UIC 0 : if (errno == EACCES || (isdir && errno == EISDIR))
227 0 : return 0;
1469 peter 228 0 : pg_log_error("could not open file \"%s\": %m", fname);
2383 peter_e 229 0 : return -1;
230 : }
231 :
232 : /*
233 : * We do what pg_flush_data() would do in the backend: prefer to use
2383 peter_e 234 ECB : * sync_file_range, but fall back to posix_fadvise. We ignore errors
235 : * because this is only a hint.
236 : */
237 : #if defined(HAVE_SYNC_FILE_RANGE)
2383 peter_e 238 GIC 4151 : (void) sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE);
239 : #elif defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
240 : (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
2383 peter_e 241 ECB : #else
242 : #error PG_FLUSH_DATA_WORKS should not have been defined
243 : #endif
244 :
2383 peter_e 245 GIC 4151 : (void) close(fd);
246 4151 : return 0;
247 : }
248 :
249 : #endif /* PG_FLUSH_DATA_WORKS */
250 :
251 : /*
252 : * fsync_fname -- Try to fsync a file or directory
253 : *
254 : * Ignores errors trying to open unreadable files, or trying to fsync
1140 peter 255 ECB : * directories on systems where that isn't allowed/required. All other errors
256 : * are fatal.
257 : */
258 : int
1469 peter 259 GIC 4199 : fsync_fname(const char *fname, bool isdir)
260 : {
261 : int fd;
262 : int flags;
263 : int returncode;
264 :
265 : /*
266 : * Some OSs require directories to be opened read-only whereas other
2383 peter_e 267 ECB : * systems don't allow us to fsync files opened read-only; so we need both
268 : * cases here. Using O_RDWR will cause us to fail to fsync files that are
269 : * not writable by our userid, but we assume that's OK.
270 : */
2383 peter_e 271 CBC 4199 : flags = PG_BINARY;
2383 peter_e 272 GIC 4199 : if (!isdir)
273 4067 : flags |= O_RDWR;
274 : else
275 132 : flags |= O_RDONLY;
276 :
277 : /*
2383 peter_e 278 ECB : * Open the file, silently ignoring errors about unreadable files (or
279 : * unsupported operations, e.g. opening a directory under Windows), and
280 : * logging others.
2383 peter_e 281 EUB : */
1668 michael 282 GBC 4199 : fd = open(fname, flags, 0);
2383 peter_e 283 4199 : if (fd < 0)
2383 peter_e 284 EUB : {
2383 peter_e 285 UIC 0 : if (errno == EACCES || (isdir && errno == EISDIR))
286 0 : return 0;
1469 peter 287 LBC 0 : pg_log_error("could not open file \"%s\": %m", fname);
2383 peter_e 288 UIC 0 : return -1;
289 : }
290 :
2383 peter_e 291 GIC 4199 : returncode = fsync(fd);
292 :
2383 peter_e 293 ECB : /*
294 : * Some OSes don't allow us to fsync directories at all, so we can ignore
2383 peter_e 295 EUB : * those errors. Anything else needs to be reported.
296 : */
1505 tmunro 297 GBC 4199 : if (returncode != 0 && !(isdir && (errno == EBADF || errno == EINVAL)))
298 : {
366 tgl 299 UIC 0 : pg_log_error("could not fsync file \"%s\": %m", fname);
2380 tgl 300 LBC 0 : (void) close(fd);
1140 peter 301 0 : exit(EXIT_FAILURE);
302 : }
303 :
2383 peter_e 304 GIC 4199 : (void) close(fd);
305 4199 : return 0;
306 : }
307 :
308 : /*
309 : * fsync_parent_path -- fsync the parent path of a file or directory
310 : *
2383 peter_e 311 ECB : * This is aimed at making file operations persistent on disk in case of
312 : * an OS crash or power failure.
313 : */
314 : int
1469 peter 315 CBC 14 : fsync_parent_path(const char *fname)
2383 peter_e 316 ECB : {
317 : char parentpath[MAXPGPATH];
318 :
2383 peter_e 319 GIC 14 : strlcpy(parentpath, fname, MAXPGPATH);
320 14 : get_parent_directory(parentpath);
321 :
322 : /*
2383 peter_e 323 ECB : * get_parent_directory() returns an empty string if the input argument is
2383 peter_e 324 EUB : * just a file name (see comments in path.c), so handle that as being the
325 : * current directory.
2383 peter_e 326 ECB : */
2383 peter_e 327 GBC 14 : if (strlen(parentpath) == 0)
2383 peter_e 328 UIC 0 : strlcpy(parentpath, ".", MAXPGPATH);
2383 peter_e 329 ECB :
1469 peter 330 GIC 14 : if (fsync_fname(parentpath, true) != 0)
2383 peter_e 331 UIC 0 : return -1;
332 :
2383 peter_e 333 GIC 14 : return 0;
334 : }
335 :
336 : /*
337 : * durable_rename -- rename(2) wrapper, issuing fsyncs required for durability
2383 peter_e 338 ECB : *
339 : * Wrapper around rename, similar to the backend version.
340 : */
341 : int
1469 peter 342 GIC 3 : durable_rename(const char *oldfile, const char *newfile)
343 : {
344 : int fd;
345 :
346 : /*
347 : * First fsync the old and target path (if it exists), to ensure that they
348 : * are properly persistent on disk. Syncing the target file is not
2383 peter_e 349 ECB : * strictly necessary, but it makes it easier to reason about crashes;
2383 peter_e 350 EUB : * because it's then guaranteed that either source or target file exists
351 : * after a crash.
2383 peter_e 352 ECB : */
1469 peter 353 CBC 3 : if (fsync_fname(oldfile, false) != 0)
2383 peter_e 354 UIC 0 : return -1;
2383 peter_e 355 ECB :
2383 peter_e 356 GIC 3 : fd = open(newfile, PG_BINARY | O_RDWR, 0);
2383 peter_e 357 GBC 3 : if (fd < 0)
2383 peter_e 358 EUB : {
2383 peter_e 359 GIC 3 : if (errno != ENOENT)
360 : {
1469 peter 361 UIC 0 : pg_log_error("could not open file \"%s\": %m", newfile);
2383 peter_e 362 0 : return -1;
2383 peter_e 363 EUB : }
364 : }
365 : else
366 : {
2383 peter_e 367 UBC 0 : if (fsync(fd) != 0)
368 : {
366 tgl 369 0 : pg_log_error("could not fsync file \"%s\": %m", newfile);
2383 peter_e 370 UIC 0 : close(fd);
1140 peter 371 0 : exit(EXIT_FAILURE);
372 : }
2383 peter_e 373 LBC 0 : close(fd);
374 : }
2383 peter_e 375 EUB :
376 : /* Time to do the real deal... */
2383 peter_e 377 GBC 3 : if (rename(oldfile, newfile) != 0)
378 : {
1469 peter 379 UIC 0 : pg_log_error("could not rename file \"%s\" to \"%s\": %m",
380 : oldfile, newfile);
2383 peter_e 381 0 : return -1;
382 : }
383 :
2383 peter_e 384 ECB : /*
2383 peter_e 385 EUB : * To guarantee renaming the file is persistent, fsync the file with its
386 : * new name, and its containing directory.
2383 peter_e 387 ECB : */
1469 peter 388 GBC 3 : if (fsync_fname(newfile, false) != 0)
2383 peter_e 389 UIC 0 : return -1;
2383 peter_e 390 ECB :
1469 peter 391 GIC 3 : if (fsync_parent_path(newfile) != 0)
2383 peter_e 392 UIC 0 : return -1;
393 :
2383 peter_e 394 GIC 3 : return 0;
395 : }
396 :
397 : #endif /* FRONTEND */
398 :
399 : /*
400 : * Return the type of a directory entry.
401 : *
944 tmunro 402 ECB : * In frontend code, elevel should be a level from logging.h; in backend code
403 : * it should be a level from elog.h.
404 : */
405 : PGFileType
944 tmunro 406 GIC 321423 : get_dirent_type(const char *path,
407 : const struct dirent *de,
408 : bool look_through_symlinks,
409 : int elevel)
410 : {
411 : PGFileType result;
412 :
413 : /*
414 : * Some systems tell us the type directly in the dirent struct, but that's
415 : * a BSD and Linux extension not required by POSIX. Even when the
944 tmunro 416 ECB : * interface is present, sometimes the type is unknown, depending on the
417 : * filesystem.
418 : */
419 : #if defined(DT_REG) && defined(DT_DIR) && defined(DT_LNK)
944 tmunro 420 CBC 321423 : if (de->d_type == DT_REG)
421 318685 : result = PGFILETYPE_REG;
944 tmunro 422 GIC 2738 : else if (de->d_type == DT_DIR)
944 tmunro 423 GBC 2719 : result = PGFILETYPE_DIR;
944 tmunro 424 GIC 19 : else if (de->d_type == DT_LNK && !look_through_symlinks)
425 19 : result = PGFILETYPE_LNK;
426 : else
944 tmunro 427 UIC 0 : result = PGFILETYPE_UNKNOWN;
944 tmunro 428 ECB : #else
429 : result = PGFILETYPE_UNKNOWN;
430 : #endif
431 :
944 tmunro 432 GIC 321423 : if (result == PGFILETYPE_UNKNOWN)
433 : {
944 tmunro 434 EUB : struct stat fst;
435 : int sret;
436 :
437 :
944 tmunro 438 UIC 0 : if (look_through_symlinks)
944 tmunro 439 UBC 0 : sret = stat(path, &fst);
440 : else
441 0 : sret = lstat(path, &fst);
442 :
443 0 : if (sret < 0)
444 : {
445 0 : result = PGFILETYPE_ERROR;
446 : #ifdef FRONTEND
366 tgl 447 UIC 0 : pg_log_generic(elevel, PG_LOG_PRIMARY, "could not stat file \"%s\": %m", path);
448 : #else
944 tmunro 449 0 : ereport(elevel,
944 tmunro 450 EUB : (errcode_for_file_access(),
451 : errmsg("could not stat file \"%s\": %m", path)));
452 : #endif
453 : }
944 tmunro 454 UBC 0 : else if (S_ISREG(fst.st_mode))
455 0 : result = PGFILETYPE_REG;
944 tmunro 456 UIC 0 : else if (S_ISDIR(fst.st_mode))
457 0 : result = PGFILETYPE_DIR;
458 0 : else if (S_ISLNK(fst.st_mode))
459 0 : result = PGFILETYPE_LNK;
460 : }
461 :
944 tmunro 462 GIC 321423 : return result;
463 : }
464 :
465 : /*
466 : * pg_pwritev_with_retry
467 : *
468 : * Convenience wrapper for pg_pwritev() that retries on partial write. If an
469 : * error is returned, it is unspecified how much has been written.
470 : */
471 : ssize_t
164 michael 472 GNC 383622 : pg_pwritev_with_retry(int fd, const struct iovec *iov, int iovcnt, off_t offset)
473 : {
474 : struct iovec iov_copy[PG_IOV_MAX];
475 383622 : ssize_t sum = 0;
476 : ssize_t part;
477 :
478 : /* We'd better have space to make a copy, in case we need to retry. */
479 383622 : if (iovcnt > PG_IOV_MAX)
480 : {
164 michael 481 UNC 0 : errno = EINVAL;
482 0 : return -1;
483 : }
484 :
485 : for (;;)
486 : {
487 : /* Write as much as we can. */
164 michael 488 GNC 383622 : part = pg_pwritev(fd, iov, iovcnt, offset);
489 383622 : if (part < 0)
164 michael 490 UNC 0 : return -1;
491 :
492 : #ifdef SIMULATE_SHORT_WRITE
493 : part = Min(part, 4096);
494 : #endif
495 :
496 : /* Count our progress. */
164 michael 497 GNC 383622 : sum += part;
498 383622 : offset += part;
499 :
500 : /* Step over iovecs that are done. */
501 2014487 : while (iovcnt > 0 && iov->iov_len <= part)
502 : {
503 1630865 : part -= iov->iov_len;
504 1630865 : ++iov;
505 1630865 : --iovcnt;
506 : }
507 :
508 : /* Are they all done? */
509 383622 : if (iovcnt == 0)
510 : {
511 : /* We don't expect the kernel to write more than requested. */
512 383622 : Assert(part == 0);
513 383622 : break;
514 : }
515 :
516 : /*
517 : * Move whatever's left to the front of our mutable copy and adjust
518 : * the leading iovec.
519 : */
164 michael 520 UNC 0 : Assert(iovcnt > 0);
521 0 : memmove(iov_copy, iov, sizeof(*iov) * iovcnt);
522 0 : Assert(iov->iov_len > part);
523 0 : iov_copy[0].iov_base = (char *) iov_copy[0].iov_base + part;
524 0 : iov_copy[0].iov_len -= part;
525 0 : iov = iov_copy;
526 : }
527 :
164 michael 528 GNC 383622 : return sum;
529 : }
530 :
531 : /*
532 : * pg_pwrite_zeros
533 : *
534 : * Writes zeros to file worth "size" bytes at "offset" (from the start of the
535 : * file), using vectored I/O.
536 : *
537 : * Returns the total amount of data written. On failure, a negative value
538 : * is returned with errno set.
539 : */
540 : ssize_t
34 541 345216 : pg_pwrite_zeros(int fd, size_t size, off_t offset)
542 : {
543 : static const PGIOAlignedBlock zbuffer = {{0}}; /* worth BLCKSZ */
1 tmunro 544 345216 : void *zerobuf_addr = unconstify(PGIOAlignedBlock *, &zbuffer)->data;
545 : struct iovec iov[PG_IOV_MAX];
34 michael 546 345216 : size_t remaining_size = size;
152 547 345216 : ssize_t total_written = 0;
548 :
549 : /* Loop, writing as many blocks as we can for each system call. */
34 550 728838 : while (remaining_size > 0)
551 : {
552 383622 : int iovcnt = 0;
553 : ssize_t written;
554 :
555 2014487 : for (; iovcnt < PG_IOV_MAX && remaining_size > 0; iovcnt++)
556 : {
557 : size_t this_iov_size;
558 :
559 1630865 : iov[iovcnt].iov_base = zerobuf_addr;
560 :
561 1630865 : if (remaining_size < BLCKSZ)
34 michael 562 UNC 0 : this_iov_size = remaining_size;
563 : else
34 michael 564 GNC 1630865 : this_iov_size = BLCKSZ;
565 :
566 1630865 : iov[iovcnt].iov_len = this_iov_size;
567 1630865 : remaining_size -= this_iov_size;
568 : }
569 :
152 570 383622 : written = pg_pwritev_with_retry(fd, iov, iovcnt, offset);
571 :
572 383622 : if (written < 0)
152 michael 573 UNC 0 : return written;
574 :
34 michael 575 GNC 383622 : offset += written;
152 576 383622 : total_written += written;
577 : }
578 :
579 345216 : Assert(total_written == size);
580 :
581 345216 : return total_written;
582 : }
|
