Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_amcheck.c
4 : * Detects corruption within database relations.
5 : *
6 : * Copyright (c) 2017-2023, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/bin/pg_amcheck/pg_amcheck.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres_fe.h"
14 :
15 : #include <limits.h>
16 : #include <time.h>
17 :
18 : #include "catalog/pg_am_d.h"
19 : #include "catalog/pg_namespace_d.h"
20 : #include "common/logging.h"
21 : #include "common/username.h"
22 : #include "fe_utils/cancel.h"
23 : #include "fe_utils/option_utils.h"
24 : #include "fe_utils/parallel_slot.h"
25 : #include "fe_utils/query_utils.h"
26 : #include "fe_utils/simple_list.h"
27 : #include "fe_utils/string_utils.h"
28 : #include "getopt_long.h" /* pgrminclude ignore */
29 : #include "pgtime.h"
30 : #include "storage/block.h"
31 :
32 : typedef struct PatternInfo
33 : {
34 : const char *pattern; /* Unaltered pattern from the command line */
35 : char *db_regex; /* Database regexp parsed from pattern, or
36 : * NULL */
37 : char *nsp_regex; /* Schema regexp parsed from pattern, or NULL */
38 : char *rel_regex; /* Relation regexp parsed from pattern, or
39 : * NULL */
40 : bool heap_only; /* true if rel_regex should only match heap
41 : * tables */
42 : bool btree_only; /* true if rel_regex should only match btree
43 : * indexes */
44 : bool matched; /* true if the pattern matched in any database */
45 : } PatternInfo;
46 :
47 : typedef struct PatternInfoArray
48 : {
49 : PatternInfo *data;
50 : size_t len;
51 : } PatternInfoArray;
52 :
53 : /* pg_amcheck command line options controlled by user flags */
54 : typedef struct AmcheckOptions
55 : {
56 : bool dbpattern;
57 : bool alldb;
58 : bool echo;
59 : bool verbose;
60 : bool strict_names;
61 : bool show_progress;
62 : int jobs;
63 :
64 : /*
65 : * Whether to install missing extensions, and optionally the name of the
66 : * schema in which to install the extension's objects.
67 : */
68 : bool install_missing;
69 : char *install_schema;
70 :
71 : /* Objects to check or not to check, as lists of PatternInfo structs. */
72 : PatternInfoArray include;
73 : PatternInfoArray exclude;
74 :
75 : /*
76 : * As an optimization, if any pattern in the exclude list applies to heap
77 : * tables, or similarly if any such pattern applies to btree indexes, or
78 : * to schemas, then these will be true, otherwise false. These should
79 : * always agree with what you'd conclude by grep'ing through the exclude
80 : * list.
81 : */
82 : bool excludetbl;
83 : bool excludeidx;
84 : bool excludensp;
85 :
86 : /*
87 : * If any inclusion pattern exists, then we should only be checking
88 : * matching relations rather than all relations, so this is true iff
89 : * include is empty.
90 : */
91 : bool allrel;
92 :
93 : /* heap table checking options */
94 : bool no_toast_expansion;
95 : bool reconcile_toast;
96 : bool on_error_stop;
97 : int64 startblock;
98 : int64 endblock;
99 : const char *skip;
100 :
101 : /* btree index checking options */
102 : bool parent_check;
103 : bool rootdescend;
104 : bool heapallindexed;
105 :
106 : /* heap and btree hybrid option */
107 : bool no_btree_expansion;
108 : } AmcheckOptions;
109 :
110 : static AmcheckOptions opts = {
111 : .dbpattern = false,
112 : .alldb = false,
113 : .echo = false,
114 : .verbose = false,
115 : .strict_names = true,
116 : .show_progress = false,
117 : .jobs = 1,
118 : .install_missing = false,
119 : .install_schema = "pg_catalog",
120 : .include = {NULL, 0},
121 : .exclude = {NULL, 0},
122 : .excludetbl = false,
123 : .excludeidx = false,
124 : .excludensp = false,
125 : .allrel = true,
126 : .no_toast_expansion = false,
127 : .reconcile_toast = true,
128 : .on_error_stop = false,
129 : .startblock = -1,
130 : .endblock = -1,
131 : .skip = "none",
132 : .parent_check = false,
133 : .rootdescend = false,
134 : .heapallindexed = false,
135 : .no_btree_expansion = false
136 : };
137 :
138 : static const char *progname = NULL;
139 :
140 : /* Whether all relations have so far passed their corruption checks */
141 : static bool all_checks_pass = true;
142 :
143 : /* Time last progress report was displayed */
144 : static pg_time_t last_progress_report = 0;
145 : static bool progress_since_last_stderr = false;
146 :
147 : typedef struct DatabaseInfo
148 : {
149 : char *datname;
150 : char *amcheck_schema; /* escaped, quoted literal */
151 : } DatabaseInfo;
152 :
153 : typedef struct RelationInfo
154 : {
155 : const DatabaseInfo *datinfo; /* shared by other relinfos */
156 : Oid reloid;
157 : bool is_heap; /* true if heap, false if btree */
158 : char *nspname;
159 : char *relname;
160 : int relpages;
161 : int blocks_to_check;
162 : char *sql; /* set during query run, pg_free'd after */
163 : } RelationInfo;
164 :
165 : /*
166 : * Query for determining if contrib's amcheck is installed. If so, selects the
167 : * namespace name where amcheck's functions can be found.
168 : */
169 : static const char *amcheck_sql =
170 : "SELECT n.nspname, x.extversion FROM pg_catalog.pg_extension x"
171 : "\nJOIN pg_catalog.pg_namespace n ON x.extnamespace = n.oid"
172 : "\nWHERE x.extname = 'amcheck'";
173 :
174 : static void prepare_heap_command(PQExpBuffer sql, RelationInfo *rel,
175 : PGconn *conn);
176 : static void prepare_btree_command(PQExpBuffer sql, RelationInfo *rel,
177 : PGconn *conn);
178 : static void run_command(ParallelSlot *slot, const char *sql);
179 : static bool verify_heap_slot_handler(PGresult *res, PGconn *conn,
180 : void *context);
181 : static bool verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context);
182 : static void help(const char *progname);
183 : static void progress_report(uint64 relations_total, uint64 relations_checked,
184 : uint64 relpages_total, uint64 relpages_checked,
185 : const char *datname, bool force, bool finished);
186 :
187 : static void append_database_pattern(PatternInfoArray *pia, const char *pattern,
188 : int encoding);
189 : static void append_schema_pattern(PatternInfoArray *pia, const char *pattern,
190 : int encoding);
191 : static void append_relation_pattern(PatternInfoArray *pia, const char *pattern,
192 : int encoding);
193 : static void append_heap_pattern(PatternInfoArray *pia, const char *pattern,
194 : int encoding);
195 : static void append_btree_pattern(PatternInfoArray *pia, const char *pattern,
196 : int encoding);
197 : static void compile_database_list(PGconn *conn, SimplePtrList *databases,
198 : const char *initial_dbname);
199 : static void compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations,
200 : const DatabaseInfo *dat,
201 : uint64 *pagecount);
202 :
203 : #define log_no_match(...) do { \
204 : if (opts.strict_names) \
205 : pg_log_error(__VA_ARGS__); \
206 : else \
207 : pg_log_warning(__VA_ARGS__); \
208 : } while(0)
209 :
210 : #define FREE_AND_SET_NULL(x) do { \
211 : pg_free(x); \
212 : (x) = NULL; \
213 : } while (0)
214 :
215 : int
758 rhaas 216 CBC 53 : main(int argc, char *argv[])
217 : {
218 53 : PGconn *conn = NULL;
219 : SimplePtrListCell *cell;
220 53 : SimplePtrList databases = {NULL, NULL};
221 53 : SimplePtrList relations = {NULL, NULL};
222 53 : bool failed = false;
223 : const char *latest_datname;
224 : int parallel_workers;
225 : ParallelSlotArray *sa;
226 : PQExpBufferData sql;
227 53 : uint64 reltotal = 0;
228 53 : uint64 pageschecked = 0;
229 53 : uint64 pagestotal = 0;
230 53 : uint64 relprogress = 0;
231 : int pattern_id;
232 :
233 : static struct option long_options[] = {
234 : /* Connection options */
235 : {"host", required_argument, NULL, 'h'},
236 : {"port", required_argument, NULL, 'p'},
237 : {"username", required_argument, NULL, 'U'},
238 : {"no-password", no_argument, NULL, 'w'},
239 : {"password", no_argument, NULL, 'W'},
240 : {"maintenance-db", required_argument, NULL, 1},
241 :
242 : /* check options */
243 : {"all", no_argument, NULL, 'a'},
244 : {"database", required_argument, NULL, 'd'},
245 : {"exclude-database", required_argument, NULL, 'D'},
246 : {"echo", no_argument, NULL, 'e'},
247 : {"index", required_argument, NULL, 'i'},
248 : {"exclude-index", required_argument, NULL, 'I'},
249 : {"jobs", required_argument, NULL, 'j'},
250 : {"progress", no_argument, NULL, 'P'},
251 : {"relation", required_argument, NULL, 'r'},
252 : {"exclude-relation", required_argument, NULL, 'R'},
253 : {"schema", required_argument, NULL, 's'},
254 : {"exclude-schema", required_argument, NULL, 'S'},
255 : {"table", required_argument, NULL, 't'},
256 : {"exclude-table", required_argument, NULL, 'T'},
257 : {"verbose", no_argument, NULL, 'v'},
258 : {"no-dependent-indexes", no_argument, NULL, 2},
259 : {"no-dependent-toast", no_argument, NULL, 3},
260 : {"exclude-toast-pointers", no_argument, NULL, 4},
261 : {"on-error-stop", no_argument, NULL, 5},
262 : {"skip", required_argument, NULL, 6},
263 : {"startblock", required_argument, NULL, 7},
264 : {"endblock", required_argument, NULL, 8},
265 : {"rootdescend", no_argument, NULL, 9},
266 : {"no-strict-names", no_argument, NULL, 10},
267 : {"heapallindexed", no_argument, NULL, 11},
268 : {"parent-check", no_argument, NULL, 12},
269 : {"install-missing", optional_argument, NULL, 13},
270 :
271 : {NULL, 0, NULL, 0}
272 : };
273 :
274 : int optindex;
275 : int c;
276 :
277 53 : const char *db = NULL;
278 53 : const char *maintenance_db = NULL;
279 :
280 53 : const char *host = NULL;
281 53 : const char *port = NULL;
282 53 : const char *username = NULL;
283 53 : enum trivalue prompt_password = TRI_DEFAULT;
284 53 : int encoding = pg_get_encoding_from_locale(NULL, false);
285 : ConnParams cparams;
286 :
287 53 : pg_logging_init(argv[0]);
288 53 : progname = get_progname(argv[0]);
289 53 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_amcheck"));
290 :
291 53 : handle_help_version_opts(argc, argv, progname, help);
292 :
293 : /* process command-line options */
118 peter 294 GNC 185 : while ((c = getopt_long(argc, argv, "ad:D:eh:Hi:I:j:p:Pr:R:s:S:t:T:U:vwW",
758 rhaas 295 CBC 185 : long_options, &optindex)) != -1)
296 : {
297 : char *endptr;
298 : unsigned long optval;
299 :
300 146 : switch (c)
301 : {
302 4 : case 'a':
303 4 : opts.alldb = true;
304 4 : break;
305 28 : case 'd':
306 28 : opts.dbpattern = true;
307 28 : append_database_pattern(&opts.include, optarg, encoding);
308 26 : break;
309 1 : case 'D':
310 1 : opts.dbpattern = true;
311 1 : append_database_pattern(&opts.exclude, optarg, encoding);
758 rhaas 312 UBC 0 : break;
313 0 : case 'e':
314 0 : opts.echo = true;
315 0 : break;
316 0 : case 'h':
317 0 : host = pg_strdup(optarg);
318 0 : break;
758 rhaas 319 CBC 9 : case 'i':
320 9 : opts.allrel = false;
321 9 : append_btree_pattern(&opts.include, optarg, encoding);
322 9 : break;
323 2 : case 'I':
324 2 : opts.excludeidx = true;
325 2 : append_btree_pattern(&opts.exclude, optarg, encoding);
326 2 : break;
758 rhaas 327 UBC 0 : case 'j':
624 michael 328 0 : if (!option_parse_int(optarg, "-j/--jobs", 1, INT_MAX,
329 : &opts.jobs))
758 rhaas 330 0 : exit(1);
331 0 : break;
758 rhaas 332 CBC 25 : case 'p':
333 25 : port = pg_strdup(optarg);
334 25 : break;
758 rhaas 335 UBC 0 : case 'P':
336 0 : opts.show_progress = true;
337 0 : break;
758 rhaas 338 CBC 7 : case 'r':
339 7 : opts.allrel = false;
340 7 : append_relation_pattern(&opts.include, optarg, encoding);
341 7 : break;
758 rhaas 342 UBC 0 : case 'R':
343 0 : opts.excludeidx = true;
344 0 : opts.excludetbl = true;
345 0 : append_relation_pattern(&opts.exclude, optarg, encoding);
346 0 : break;
758 rhaas 347 CBC 19 : case 's':
348 19 : opts.allrel = false;
349 19 : append_schema_pattern(&opts.include, optarg, encoding);
350 17 : break;
351 7 : case 'S':
352 7 : opts.excludensp = true;
353 7 : append_schema_pattern(&opts.exclude, optarg, encoding);
354 6 : break;
355 15 : case 't':
356 15 : opts.allrel = false;
357 15 : append_heap_pattern(&opts.include, optarg, encoding);
358 13 : break;
359 3 : case 'T':
360 3 : opts.excludetbl = true;
361 3 : append_heap_pattern(&opts.exclude, optarg, encoding);
362 2 : break;
363 1 : case 'U':
364 1 : username = pg_strdup(optarg);
365 1 : break;
118 peter 366 UNC 0 : case 'v':
367 0 : opts.verbose = true;
368 0 : pg_logging_increase_verbosity();
369 0 : break;
758 rhaas 370 UBC 0 : case 'w':
371 0 : prompt_password = TRI_NO;
372 0 : break;
373 0 : case 'W':
374 0 : prompt_password = TRI_YES;
375 0 : break;
376 0 : case 1:
377 0 : maintenance_db = pg_strdup(optarg);
378 0 : break;
758 rhaas 379 CBC 4 : case 2:
380 4 : opts.no_btree_expansion = true;
381 4 : break;
382 1 : case 3:
383 1 : opts.no_toast_expansion = true;
384 1 : break;
385 1 : case 4:
386 1 : opts.reconcile_toast = false;
387 1 : break;
758 rhaas 388 UBC 0 : case 5:
389 0 : opts.on_error_stop = true;
390 0 : break;
391 0 : case 6:
392 0 : if (pg_strcasecmp(optarg, "all-visible") == 0)
599 dgustafsson 393 0 : opts.skip = "all-visible";
758 rhaas 394 0 : else if (pg_strcasecmp(optarg, "all-frozen") == 0)
599 dgustafsson 395 0 : opts.skip = "all-frozen";
396 0 : else if (pg_strcasecmp(optarg, "none") == 0)
397 0 : opts.skip = "none";
398 : else
366 tgl 399 0 : pg_fatal("invalid argument for option %s", "--skip");
758 rhaas 400 0 : break;
758 rhaas 401 CBC 2 : case 7:
597 peter 402 2 : errno = 0;
403 2 : optval = strtoul(optarg, &endptr, 10);
404 2 : if (endptr == optarg || *endptr != '\0' || errno != 0)
366 tgl 405 1 : pg_fatal("invalid start block");
597 peter 406 1 : if (optval > MaxBlockNumber)
366 tgl 407 UBC 0 : pg_fatal("start block out of bounds");
597 peter 408 CBC 1 : opts.startblock = optval;
758 rhaas 409 1 : break;
410 2 : case 8:
597 peter 411 2 : errno = 0;
412 2 : optval = strtoul(optarg, &endptr, 10);
413 2 : if (endptr == optarg || *endptr != '\0' || errno != 0)
366 tgl 414 1 : pg_fatal("invalid end block");
597 peter 415 1 : if (optval > MaxBlockNumber)
366 tgl 416 UBC 0 : pg_fatal("end block out of bounds");
597 peter 417 CBC 1 : opts.endblock = optval;
758 rhaas 418 1 : break;
419 1 : case 9:
420 1 : opts.rootdescend = true;
421 1 : opts.parent_check = true;
422 1 : break;
423 11 : case 10:
424 11 : opts.strict_names = false;
425 11 : break;
426 1 : case 11:
427 1 : opts.heapallindexed = true;
428 1 : break;
429 1 : case 12:
430 1 : opts.parent_check = true;
431 1 : break;
715 andrew 432 UBC 0 : case 13:
433 0 : opts.install_missing = true;
434 0 : if (optarg)
435 0 : opts.install_schema = pg_strdup(optarg);
436 0 : break;
758 rhaas 437 CBC 1 : default:
438 : /* getopt_long already emitted a complaint */
366 tgl 439 1 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
758 rhaas 440 1 : exit(1);
441 : }
442 : }
443 :
444 39 : if (opts.endblock >= 0 && opts.endblock < opts.startblock)
366 tgl 445 1 : pg_fatal("end block precedes start block");
446 :
447 : /*
448 : * A single non-option arguments specifies a database name or connection
449 : * string.
450 : */
758 rhaas 451 38 : if (optind < argc)
452 : {
453 20 : db = argv[optind];
454 20 : optind++;
455 : }
456 :
457 38 : if (optind < argc)
458 : {
758 rhaas 459 UBC 0 : pg_log_error("too many command-line arguments (first is \"%s\")",
460 : argv[optind]);
366 tgl 461 0 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
758 rhaas 462 0 : exit(1);
463 : }
464 :
465 : /* fill cparams except for dbname, which is set below */
758 rhaas 466 CBC 38 : cparams.pghost = host;
467 38 : cparams.pgport = port;
468 38 : cparams.pguser = username;
469 38 : cparams.prompt_password = prompt_password;
470 38 : cparams.dbname = NULL;
471 38 : cparams.override_dbname = NULL;
472 :
473 38 : setup_cancel_handler(NULL);
474 :
475 : /* choose the database for our initial connection */
476 38 : if (opts.alldb)
477 : {
478 4 : if (db != NULL)
366 tgl 479 UBC 0 : pg_fatal("cannot specify a database name with --all");
758 rhaas 480 CBC 4 : cparams.dbname = maintenance_db;
481 : }
482 34 : else if (db != NULL)
483 : {
484 20 : if (opts.dbpattern)
366 tgl 485 UBC 0 : pg_fatal("cannot specify both a database name and database patterns");
758 rhaas 486 CBC 20 : cparams.dbname = db;
487 : }
488 :
489 38 : if (opts.alldb || opts.dbpattern)
490 : {
491 18 : conn = connectMaintenanceDatabase(&cparams, progname, opts.echo);
492 18 : compile_database_list(conn, &databases, NULL);
493 : }
494 : else
495 : {
496 20 : if (cparams.dbname == NULL)
497 : {
758 rhaas 498 UBC 0 : if (getenv("PGDATABASE"))
499 0 : cparams.dbname = getenv("PGDATABASE");
500 0 : else if (getenv("PGUSER"))
501 0 : cparams.dbname = getenv("PGUSER");
502 : else
503 0 : cparams.dbname = get_user_name_or_exit(progname);
504 : }
758 rhaas 505 CBC 20 : conn = connectDatabase(&cparams, progname, opts.echo, false, true);
506 18 : compile_database_list(conn, &databases, PQdb(conn));
507 : }
508 :
509 31 : if (databases.head == NULL)
510 : {
758 rhaas 511 UBC 0 : if (conn != NULL)
512 0 : disconnectDatabase(conn);
366 tgl 513 0 : pg_log_warning("no databases to check");
758 rhaas 514 0 : exit(0);
515 : }
516 :
517 : /*
518 : * Compile a list of all relations spanning all databases to be checked.
519 : */
758 rhaas 520 CBC 80 : for (cell = databases.head; cell; cell = cell->next)
521 : {
522 : PGresult *result;
523 : int ntups;
524 49 : const char *amcheck_schema = NULL;
525 49 : DatabaseInfo *dat = (DatabaseInfo *) cell->ptr;
526 :
527 49 : cparams.override_dbname = dat->datname;
528 49 : if (conn == NULL || strcmp(PQdb(conn), dat->datname) != 0)
529 : {
530 26 : if (conn != NULL)
531 22 : disconnectDatabase(conn);
532 26 : conn = connectDatabase(&cparams, progname, opts.echo, false, true);
533 : }
534 :
535 : /*
536 : * Optionally install amcheck if not already installed in this
537 : * database.
538 : */
715 andrew 539 49 : if (opts.install_missing)
540 : {
541 : char *schema;
542 : char *install_sql;
543 :
544 : /*
545 : * Must re-escape the schema name for each database, as the
546 : * escaping rules may change.
547 : */
715 andrew 548 UBC 0 : schema = PQescapeIdentifier(conn, opts.install_schema,
549 0 : strlen(opts.install_schema));
550 0 : install_sql = psprintf("CREATE EXTENSION IF NOT EXISTS amcheck WITH SCHEMA %s",
551 : schema);
552 :
553 0 : executeCommand(conn, install_sql, opts.echo);
554 0 : pfree(install_sql);
555 0 : pfree(schema);
556 : }
557 :
558 : /*
559 : * Verify that amcheck is installed for this next database. User
560 : * error could result in a database not having amcheck that should
561 : * have it, but we also could be iterating over multiple databases
562 : * where not all of them have amcheck installed (for example,
563 : * 'template1').
564 : */
758 rhaas 565 CBC 49 : result = executeQuery(conn, amcheck_sql, opts.echo);
566 49 : if (PQresultStatus(result) != PGRES_TUPLES_OK)
567 : {
568 : /* Querying the catalog failed. */
758 rhaas 569 UBC 0 : pg_log_error("database \"%s\": %s",
570 : PQdb(conn), PQerrorMessage(conn));
366 tgl 571 0 : pg_log_error_detail("Query was: %s", amcheck_sql);
758 rhaas 572 0 : PQclear(result);
573 0 : disconnectDatabase(conn);
574 0 : exit(1);
575 : }
758 rhaas 576 CBC 49 : ntups = PQntuples(result);
577 49 : if (ntups == 0)
578 : {
579 : /* Querying the catalog succeeded, but amcheck is missing. */
580 11 : pg_log_warning("skipping database \"%s\": amcheck is not installed",
581 : PQdb(conn));
582 11 : disconnectDatabase(conn);
583 11 : conn = NULL;
584 11 : continue;
585 : }
586 38 : amcheck_schema = PQgetvalue(result, 0, 0);
587 38 : if (opts.verbose)
758 rhaas 588 UBC 0 : pg_log_info("in database \"%s\": using amcheck version \"%s\" in schema \"%s\"",
589 : PQdb(conn), PQgetvalue(result, 0, 1), amcheck_schema);
758 rhaas 590 CBC 38 : dat->amcheck_schema = PQescapeIdentifier(conn, amcheck_schema,
591 : strlen(amcheck_schema));
592 38 : PQclear(result);
593 :
594 38 : compile_relation_list_one_db(conn, &relations, dat, &pagestotal);
595 : }
596 :
597 : /*
598 : * Check that all inclusion patterns matched at least one schema or
599 : * relation that we can check.
600 : */
601 90 : for (pattern_id = 0; pattern_id < opts.include.len; pattern_id++)
602 : {
603 59 : PatternInfo *pat = &opts.include.data[pattern_id];
604 :
605 59 : if (!pat->matched && (pat->nsp_regex != NULL || pat->rel_regex != NULL))
606 : {
607 23 : failed = opts.strict_names;
608 :
597 dgustafsson 609 23 : if (pat->heap_only)
610 7 : log_no_match("no heap tables to check matching \"%s\"",
611 : pat->pattern);
612 16 : else if (pat->btree_only)
613 5 : log_no_match("no btree indexes to check matching \"%s\"",
614 : pat->pattern);
615 11 : else if (pat->rel_regex == NULL)
616 4 : log_no_match("no relations to check in schemas matching \"%s\"",
617 : pat->pattern);
618 : else
619 7 : log_no_match("no relations to check matching \"%s\"",
620 : pat->pattern);
621 : }
622 : }
623 :
758 rhaas 624 31 : if (failed)
625 : {
626 1 : if (conn != NULL)
627 1 : disconnectDatabase(conn);
628 1 : exit(1);
629 : }
630 :
631 : /*
632 : * Set parallel_workers to the lesser of opts.jobs and the number of
633 : * relations.
634 : */
635 30 : parallel_workers = 0;
636 5858 : for (cell = relations.head; cell; cell = cell->next)
637 : {
638 5828 : reltotal++;
639 5828 : if (parallel_workers < opts.jobs)
640 26 : parallel_workers++;
641 : }
642 :
643 30 : if (reltotal == 0)
644 : {
645 4 : if (conn != NULL)
758 rhaas 646 UBC 0 : disconnectDatabase(conn);
366 tgl 647 CBC 4 : pg_fatal("no relations to check");
648 : }
758 rhaas 649 26 : progress_report(reltotal, relprogress, pagestotal, pageschecked,
650 : NULL, true, false);
651 :
652 : /*
653 : * Main event loop.
654 : *
655 : * We use server-side parallelism to check up to parallel_workers
656 : * relations in parallel. The list of relations was computed in database
657 : * order, which minimizes the number of connects and disconnects as we
658 : * process the list.
659 : */
660 26 : latest_datname = NULL;
661 26 : sa = ParallelSlotsSetup(parallel_workers, &cparams, progname, opts.echo,
662 : NULL);
663 26 : if (conn != NULL)
664 : {
665 23 : ParallelSlotsAdoptConn(sa, conn);
666 23 : conn = NULL;
667 : }
668 :
669 26 : initPQExpBuffer(&sql);
670 5854 : for (relprogress = 0, cell = relations.head; cell; cell = cell->next)
671 : {
672 : ParallelSlot *free_slot;
673 : RelationInfo *rel;
674 :
675 5828 : rel = (RelationInfo *) cell->ptr;
676 :
677 5828 : if (CancelRequested)
678 : {
758 rhaas 679 UBC 0 : failed = true;
680 0 : break;
681 : }
682 :
683 : /*
684 : * The list of relations is in database sorted order. If this next
685 : * relation is in a different database than the last one seen, we are
686 : * about to start checking this database. Note that other slots may
687 : * still be working on relations from prior databases.
688 : */
758 rhaas 689 CBC 5828 : latest_datname = rel->datinfo->datname;
690 :
691 5828 : progress_report(reltotal, relprogress, pagestotal, pageschecked,
692 : latest_datname, false, false);
693 :
694 5828 : relprogress++;
695 5828 : pageschecked += rel->blocks_to_check;
696 :
697 : /*
698 : * Get a parallel slot for the next amcheck command, blocking if
699 : * necessary until one is available, or until a previously issued slot
700 : * command fails, indicating that we should abort checking the
701 : * remaining objects.
702 : */
703 5828 : free_slot = ParallelSlotsGetIdle(sa, rel->datinfo->datname);
704 5828 : if (!free_slot)
705 : {
706 : /*
707 : * Something failed. We don't need to know what it was, because
708 : * the handler should already have emitted the necessary error
709 : * messages.
710 : */
758 rhaas 711 UBC 0 : failed = true;
712 0 : break;
713 : }
714 :
758 rhaas 715 CBC 5828 : if (opts.verbose)
758 rhaas 716 UBC 0 : PQsetErrorVerbosity(free_slot->connection, PQERRORS_VERBOSE);
717 :
718 : /*
719 : * Execute the appropriate amcheck command for this relation using our
720 : * slot's database connection. We do not wait for the command to
721 : * complete, nor do we perform any error checking, as that is done by
722 : * the parallel slots and our handler callback functions.
723 : */
758 rhaas 724 CBC 5828 : if (rel->is_heap)
725 : {
726 2671 : if (opts.verbose)
727 : {
758 rhaas 728 UBC 0 : if (opts.show_progress && progress_since_last_stderr)
729 0 : fprintf(stderr, "\n");
610 peter 730 0 : pg_log_info("checking heap table \"%s.%s.%s\"",
731 : rel->datinfo->datname, rel->nspname, rel->relname);
758 rhaas 732 0 : progress_since_last_stderr = false;
733 : }
758 rhaas 734 CBC 2671 : prepare_heap_command(&sql, rel, free_slot->connection);
735 2671 : rel->sql = pstrdup(sql.data); /* pg_free'd after command */
736 2671 : ParallelSlotSetHandler(free_slot, verify_heap_slot_handler, rel);
737 2671 : run_command(free_slot, rel->sql);
738 : }
739 : else
740 : {
741 3157 : if (opts.verbose)
742 : {
758 rhaas 743 UBC 0 : if (opts.show_progress && progress_since_last_stderr)
744 0 : fprintf(stderr, "\n");
745 :
610 peter 746 0 : pg_log_info("checking btree index \"%s.%s.%s\"",
747 : rel->datinfo->datname, rel->nspname, rel->relname);
758 rhaas 748 0 : progress_since_last_stderr = false;
749 : }
758 rhaas 750 CBC 3157 : prepare_btree_command(&sql, rel, free_slot->connection);
751 3157 : rel->sql = pstrdup(sql.data); /* pg_free'd after command */
752 3157 : ParallelSlotSetHandler(free_slot, verify_btree_slot_handler, rel);
753 3157 : run_command(free_slot, rel->sql);
754 : }
755 : }
756 26 : termPQExpBuffer(&sql);
757 :
758 26 : if (!failed)
759 : {
760 :
761 : /*
762 : * Wait for all slots to complete, or for one to indicate that an
763 : * error occurred. Like above, we rely on the handler emitting the
764 : * necessary error messages.
765 : */
766 26 : if (sa && !ParallelSlotsWaitCompletion(sa))
758 rhaas 767 UBC 0 : failed = true;
768 :
758 rhaas 769 CBC 26 : progress_report(reltotal, relprogress, pagestotal, pageschecked, NULL, true, true);
770 : }
771 :
772 26 : if (sa)
773 : {
774 26 : ParallelSlotsTerminate(sa);
775 26 : FREE_AND_SET_NULL(sa);
776 : }
777 :
778 26 : if (failed)
758 rhaas 779 UBC 0 : exit(1);
780 :
758 rhaas 781 CBC 26 : if (!all_checks_pass)
782 11 : exit(2);
783 : }
784 :
785 : /*
786 : * prepare_heap_command
787 : *
788 : * Creates a SQL command for running amcheck checking on the given heap
789 : * relation. The command is phrased as a SQL query, with column order and
790 : * names matching the expectations of verify_heap_slot_handler, which will
791 : * receive and handle each row returned from the verify_heapam() function.
792 : *
793 : * The constructed SQL command will silently skip temporary tables, as checking
794 : * them would needlessly draw errors from the underlying amcheck function.
795 : *
796 : * sql: buffer into which the heap table checking command will be written
797 : * rel: relation information for the heap table to be checked
798 : * conn: the connection to be used, for string escaping purposes
799 : */
800 : static void
801 2671 : prepare_heap_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
802 : {
803 2671 : resetPQExpBuffer(sql);
804 5342 : appendPQExpBuffer(sql,
805 : "SELECT v.blkno, v.offnum, v.attnum, v.msg "
806 : "FROM pg_catalog.pg_class c, %s.verify_heapam("
807 : "\nrelation := c.oid, on_error_stop := %s, check_toast := %s, skip := '%s'",
808 2671 : rel->datinfo->amcheck_schema,
809 2671 : opts.on_error_stop ? "true" : "false",
810 2671 : opts.reconcile_toast ? "true" : "false",
811 : opts.skip);
812 :
813 2671 : if (opts.startblock >= 0)
758 rhaas 814 UBC 0 : appendPQExpBuffer(sql, ", startblock := " INT64_FORMAT, opts.startblock);
758 rhaas 815 CBC 2671 : if (opts.endblock >= 0)
758 rhaas 816 UBC 0 : appendPQExpBuffer(sql, ", endblock := " INT64_FORMAT, opts.endblock);
817 :
543 pg 818 CBC 2671 : appendPQExpBuffer(sql,
819 : "\n) v WHERE c.oid = %u "
820 : "AND c.relpersistence != 't'",
821 : rel->reloid);
758 rhaas 822 2671 : }
823 :
824 : /*
825 : * prepare_btree_command
826 : *
827 : * Creates a SQL command for running amcheck checking on the given btree index
828 : * relation. The command does not select any columns, as btree checking
829 : * functions do not return any, but rather return corruption information by
830 : * raising errors, which verify_btree_slot_handler expects.
831 : *
832 : * The constructed SQL command will silently skip temporary indexes, and
833 : * indexes being reindexed concurrently, as checking them would needlessly draw
834 : * errors from the underlying amcheck functions.
835 : *
836 : * sql: buffer into which the heap table checking command will be written
837 : * rel: relation information for the index to be checked
838 : * conn: the connection to be used, for string escaping purposes
839 : */
840 : static void
841 3157 : prepare_btree_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
842 : {
843 3157 : resetPQExpBuffer(sql);
844 :
845 3157 : if (opts.parent_check)
846 40 : appendPQExpBuffer(sql,
847 : "SELECT %s.bt_index_parent_check("
848 : "index := c.oid, heapallindexed := %s, rootdescend := %s)"
849 : "\nFROM pg_catalog.pg_class c, pg_catalog.pg_index i "
850 : "WHERE c.oid = %u "
851 : "AND c.oid = i.indexrelid "
852 : "AND c.relpersistence != 't' "
853 : "AND i.indisready AND i.indisvalid AND i.indislive",
854 20 : rel->datinfo->amcheck_schema,
855 20 : (opts.heapallindexed ? "true" : "false"),
543 pg 856 20 : (opts.rootdescend ? "true" : "false"),
857 : rel->reloid);
858 : else
758 rhaas 859 3137 : appendPQExpBuffer(sql,
860 : "SELECT %s.bt_index_check("
861 : "index := c.oid, heapallindexed := %s)"
862 : "\nFROM pg_catalog.pg_class c, pg_catalog.pg_index i "
863 : "WHERE c.oid = %u "
864 : "AND c.oid = i.indexrelid "
865 : "AND c.relpersistence != 't' "
866 : "AND i.indisready AND i.indisvalid AND i.indislive",
867 3137 : rel->datinfo->amcheck_schema,
543 pg 868 3137 : (opts.heapallindexed ? "true" : "false"),
869 : rel->reloid);
758 rhaas 870 3157 : }
871 :
872 : /*
873 : * run_command
874 : *
875 : * Sends a command to the server without waiting for the command to complete.
876 : * Logs an error if the command cannot be sent, but otherwise any errors are
877 : * expected to be handled by a ParallelSlotHandler.
878 : *
879 : * If reconnecting to the database is necessary, the cparams argument may be
880 : * modified.
881 : *
882 : * slot: slot with connection to the server we should use for the command
883 : * sql: query to send
884 : */
885 : static void
886 5828 : run_command(ParallelSlot *slot, const char *sql)
887 : {
888 5828 : if (opts.echo)
758 rhaas 889 UBC 0 : printf("%s\n", sql);
890 :
758 rhaas 891 CBC 5828 : if (PQsendQuery(slot->connection, sql) == 0)
892 : {
758 rhaas 893 UBC 0 : pg_log_error("error sending command to database \"%s\": %s",
894 : PQdb(slot->connection),
895 : PQerrorMessage(slot->connection));
366 tgl 896 0 : pg_log_error_detail("Command was: %s", sql);
758 rhaas 897 0 : exit(1);
898 : }
758 rhaas 899 CBC 5828 : }
900 :
901 : /*
902 : * should_processing_continue
903 : *
904 : * Checks a query result returned from a query (presumably issued on a slot's
905 : * connection) to determine if parallel slots should continue issuing further
906 : * commands.
907 : *
908 : * Note: Heap relation corruption is reported by verify_heapam() via the result
909 : * set, rather than an ERROR, but running verify_heapam() on a corrupted heap
910 : * table may still result in an error being returned from the server due to
911 : * missing relation files, bad checksums, etc. The btree corruption checking
912 : * functions always use errors to communicate corruption messages. We can't
913 : * just abort processing because we got a mere ERROR.
914 : *
915 : * res: result from an executed sql query
916 : */
917 : static bool
918 5828 : should_processing_continue(PGresult *res)
919 : {
920 : const char *severity;
921 :
922 5828 : switch (PQresultStatus(res))
923 : {
924 : /* These are expected and ok */
925 5782 : case PGRES_COMMAND_OK:
926 : case PGRES_TUPLES_OK:
927 : case PGRES_NONFATAL_ERROR:
928 5782 : break;
929 :
930 : /* This is expected but requires closer scrutiny */
931 46 : case PGRES_FATAL_ERROR:
932 46 : severity = PQresultErrorField(res, PG_DIAG_SEVERITY_NONLOCALIZED);
307 tgl 933 46 : if (severity == NULL)
307 tgl 934 UBC 0 : return false; /* libpq failure, probably lost connection */
758 rhaas 935 CBC 46 : if (strcmp(severity, "FATAL") == 0)
758 rhaas 936 UBC 0 : return false;
758 rhaas 937 CBC 46 : if (strcmp(severity, "PANIC") == 0)
758 rhaas 938 UBC 0 : return false;
758 rhaas 939 CBC 46 : break;
940 :
941 : /* These are unexpected */
758 rhaas 942 UBC 0 : case PGRES_BAD_RESPONSE:
943 : case PGRES_EMPTY_QUERY:
944 : case PGRES_COPY_OUT:
945 : case PGRES_COPY_IN:
946 : case PGRES_COPY_BOTH:
947 : case PGRES_SINGLE_TUPLE:
948 : case PGRES_PIPELINE_SYNC:
949 : case PGRES_PIPELINE_ABORTED:
950 0 : return false;
951 : }
758 rhaas 952 CBC 5828 : return true;
953 : }
954 :
955 : /*
956 : * Returns a copy of the argument string with all lines indented four spaces.
957 : *
958 : * The caller should pg_free the result when finished with it.
959 : */
960 : static char *
961 46 : indent_lines(const char *str)
962 : {
963 : PQExpBufferData buf;
964 : const char *c;
965 : char *result;
966 :
967 46 : initPQExpBuffer(&buf);
968 46 : appendPQExpBufferStr(&buf, " ");
969 3618 : for (c = str; *c; c++)
970 : {
971 3572 : appendPQExpBufferChar(&buf, *c);
972 3572 : if (c[0] == '\n' && c[1] != '\0')
973 1 : appendPQExpBufferStr(&buf, " ");
974 : }
975 46 : result = pstrdup(buf.data);
976 46 : termPQExpBuffer(&buf);
977 :
978 46 : return result;
979 : }
980 :
981 : /*
982 : * verify_heap_slot_handler
983 : *
984 : * ParallelSlotHandler that receives results from a heap table checking command
985 : * created by prepare_heap_command and outputs the results for the user.
986 : *
987 : * res: result from an executed sql query
988 : * conn: connection on which the sql query was executed
989 : * context: the sql query being handled, as a cstring
990 : */
991 : static bool
992 2671 : verify_heap_slot_handler(PGresult *res, PGconn *conn, void *context)
993 : {
994 2671 : RelationInfo *rel = (RelationInfo *) context;
995 :
996 2671 : if (PQresultStatus(res) == PGRES_TUPLES_OK)
997 : {
998 : int i;
999 2651 : int ntups = PQntuples(res);
1000 :
1001 2651 : if (ntups > 0)
1002 9 : all_checks_pass = false;
1003 :
1004 2701 : for (i = 0; i < ntups; i++)
1005 : {
1006 : const char *msg;
1007 :
1008 : /* The message string should never be null, but check */
1009 50 : if (PQgetisnull(res, i, 3))
758 rhaas 1010 UBC 0 : msg = "NO MESSAGE";
1011 : else
758 rhaas 1012 CBC 50 : msg = PQgetvalue(res, i, 3);
1013 :
1014 50 : if (!PQgetisnull(res, i, 2))
604 peter 1015 2 : printf(_("heap table \"%s.%s.%s\", block %s, offset %s, attribute %s:\n"),
1016 : rel->datinfo->datname, rel->nspname, rel->relname,
1017 : PQgetvalue(res, i, 0), /* blkno */
1018 : PQgetvalue(res, i, 1), /* offnum */
1019 : PQgetvalue(res, i, 2)); /* attnum */
1020 :
758 rhaas 1021 48 : else if (!PQgetisnull(res, i, 1))
604 peter 1022 48 : printf(_("heap table \"%s.%s.%s\", block %s, offset %s:\n"),
1023 : rel->datinfo->datname, rel->nspname, rel->relname,
1024 : PQgetvalue(res, i, 0), /* blkno */
1025 : PQgetvalue(res, i, 1)); /* offnum */
1026 :
758 rhaas 1027 UBC 0 : else if (!PQgetisnull(res, i, 0))
604 peter 1028 0 : printf(_("heap table \"%s.%s.%s\", block %s:\n"),
1029 : rel->datinfo->datname, rel->nspname, rel->relname,
1030 : PQgetvalue(res, i, 0)); /* blkno */
1031 :
1032 : else
1033 0 : printf(_("heap table \"%s.%s.%s\":\n"),
1034 : rel->datinfo->datname, rel->nspname, rel->relname);
1035 :
604 peter 1036 CBC 50 : printf(" %s\n", msg);
1037 : }
1038 : }
758 rhaas 1039 20 : else if (PQresultStatus(res) != PGRES_TUPLES_OK)
1040 : {
1041 20 : char *msg = indent_lines(PQerrorMessage(conn));
1042 :
1043 20 : all_checks_pass = false;
604 peter 1044 20 : printf(_("heap table \"%s.%s.%s\":\n"),
1045 : rel->datinfo->datname, rel->nspname, rel->relname);
1046 20 : printf("%s", msg);
758 rhaas 1047 20 : if (opts.verbose)
610 peter 1048 UBC 0 : printf(_("query was: %s\n"), rel->sql);
758 rhaas 1049 CBC 20 : FREE_AND_SET_NULL(msg);
1050 : }
1051 :
1052 2671 : FREE_AND_SET_NULL(rel->sql);
1053 2671 : FREE_AND_SET_NULL(rel->nspname);
1054 2671 : FREE_AND_SET_NULL(rel->relname);
1055 :
1056 2671 : return should_processing_continue(res);
1057 : }
1058 :
1059 : /*
1060 : * verify_btree_slot_handler
1061 : *
1062 : * ParallelSlotHandler that receives results from a btree checking command
1063 : * created by prepare_btree_command and outputs them for the user. The results
1064 : * from the btree checking command is assumed to be empty, but when the results
1065 : * are an error code, the useful information about the corruption is expected
1066 : * in the connection's error message.
1067 : *
1068 : * res: result from an executed sql query
1069 : * conn: connection on which the sql query was executed
1070 : * context: unused
1071 : */
1072 : static bool
1073 3157 : verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context)
1074 : {
1075 3157 : RelationInfo *rel = (RelationInfo *) context;
1076 :
1077 3157 : if (PQresultStatus(res) == PGRES_TUPLES_OK)
1078 : {
332 tgl 1079 3131 : int ntups = PQntuples(res);
1080 :
543 pg 1081 3131 : if (ntups > 1)
1082 : {
1083 : /*
1084 : * We expect the btree checking functions to return one void row
1085 : * each, or zero rows if the check was skipped due to the object
1086 : * being in the wrong state to be checked, so we should output
1087 : * some sort of warning if we get anything more, not because it
1088 : * indicates corruption, but because it suggests a mismatch
1089 : * between amcheck and pg_amcheck versions.
1090 : *
1091 : * In conjunction with --progress, anything written to stderr at
1092 : * this time would present strangely to the user without an extra
1093 : * newline, so we print one. If we were multithreaded, we'd have
1094 : * to avoid splitting this across multiple calls, but we're in an
1095 : * event loop, so it doesn't matter.
1096 : */
758 rhaas 1097 UBC 0 : if (opts.show_progress && progress_since_last_stderr)
1098 0 : fprintf(stderr, "\n");
610 peter 1099 0 : pg_log_warning("btree index \"%s.%s.%s\": btree checking function returned unexpected number of rows: %d",
1100 : rel->datinfo->datname, rel->nspname, rel->relname, ntups);
758 rhaas 1101 0 : if (opts.verbose)
366 tgl 1102 0 : pg_log_warning_detail("Query was: %s", rel->sql);
1103 0 : pg_log_warning_hint("Are %s's and amcheck's versions compatible?",
1104 : progname);
758 rhaas 1105 0 : progress_since_last_stderr = false;
1106 : }
1107 : }
1108 : else
1109 : {
758 rhaas 1110 CBC 26 : char *msg = indent_lines(PQerrorMessage(conn));
1111 :
1112 26 : all_checks_pass = false;
604 peter 1113 26 : printf(_("btree index \"%s.%s.%s\":\n"),
1114 : rel->datinfo->datname, rel->nspname, rel->relname);
1115 26 : printf("%s", msg);
758 rhaas 1116 26 : if (opts.verbose)
610 peter 1117 UBC 0 : printf(_("query was: %s\n"), rel->sql);
758 rhaas 1118 CBC 26 : FREE_AND_SET_NULL(msg);
1119 : }
1120 :
1121 3157 : FREE_AND_SET_NULL(rel->sql);
1122 3157 : FREE_AND_SET_NULL(rel->nspname);
1123 3157 : FREE_AND_SET_NULL(rel->relname);
1124 :
1125 3157 : return should_processing_continue(res);
1126 : }
1127 :
1128 : /*
1129 : * help
1130 : *
1131 : * Prints help page for the program
1132 : *
1133 : * progname: the name of the executed program, such as "pg_amcheck"
1134 : */
1135 : static void
1136 1 : help(const char *progname)
1137 : {
696 peter 1138 1 : printf(_("%s checks objects in a PostgreSQL database for corruption.\n\n"), progname);
727 1139 1 : printf(_("Usage:\n"));
1140 1 : printf(_(" %s [OPTION]... [DBNAME]\n"), progname);
696 1141 1 : printf(_("\nTarget options:\n"));
1142 1 : printf(_(" -a, --all check all databases\n"));
1143 1 : printf(_(" -d, --database=PATTERN check matching database(s)\n"));
1144 1 : printf(_(" -D, --exclude-database=PATTERN do NOT check matching database(s)\n"));
1145 1 : printf(_(" -i, --index=PATTERN check matching index(es)\n"));
1146 1 : printf(_(" -I, --exclude-index=PATTERN do NOT check matching index(es)\n"));
1147 1 : printf(_(" -r, --relation=PATTERN check matching relation(s)\n"));
1148 1 : printf(_(" -R, --exclude-relation=PATTERN do NOT check matching relation(s)\n"));
1149 1 : printf(_(" -s, --schema=PATTERN check matching schema(s)\n"));
1150 1 : printf(_(" -S, --exclude-schema=PATTERN do NOT check matching schema(s)\n"));
1151 1 : printf(_(" -t, --table=PATTERN check matching table(s)\n"));
1152 1 : printf(_(" -T, --exclude-table=PATTERN do NOT check matching table(s)\n"));
1153 1 : printf(_(" --no-dependent-indexes do NOT expand list of relations to include indexes\n"));
1154 1 : printf(_(" --no-dependent-toast do NOT expand list of relations to include TOAST tables\n"));
1155 1 : printf(_(" --no-strict-names do NOT require patterns to match objects\n"));
1156 1 : printf(_("\nTable checking options:\n"));
1157 1 : printf(_(" --exclude-toast-pointers do NOT follow relation TOAST pointers\n"));
1158 1 : printf(_(" --on-error-stop stop checking at end of first corrupt page\n"));
1159 1 : printf(_(" --skip=OPTION do NOT check \"all-frozen\" or \"all-visible\" blocks\n"));
1160 1 : printf(_(" --startblock=BLOCK begin checking table(s) at the given block number\n"));
1161 1 : printf(_(" --endblock=BLOCK check table(s) only up to the given block number\n"));
1162 1 : printf(_("\nB-tree index checking options:\n"));
610 1163 1 : printf(_(" --heapallindexed check that all heap tuples are found within indexes\n"));
696 1164 1 : printf(_(" --parent-check check index parent/child relationships\n"));
1165 1 : printf(_(" --rootdescend search from root page to refind tuples\n"));
727 1166 1 : printf(_("\nConnection options:\n"));
696 1167 1 : printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
1168 1 : printf(_(" -p, --port=PORT database server port\n"));
1169 1 : printf(_(" -U, --username=USERNAME user name to connect as\n"));
1170 1 : printf(_(" -w, --no-password never prompt for password\n"));
1171 1 : printf(_(" -W, --password force password prompt\n"));
1172 1 : printf(_(" --maintenance-db=DBNAME alternate maintenance database\n"));
1173 1 : printf(_("\nOther options:\n"));
1174 1 : printf(_(" -e, --echo show the commands being sent to the server\n"));
1175 1 : printf(_(" -j, --jobs=NUM use this many concurrent connections to the server\n"));
675 1176 1 : printf(_(" -P, --progress show progress information\n"));
696 1177 1 : printf(_(" -v, --verbose write a lot of output\n"));
1178 1 : printf(_(" -V, --version output version information, then exit\n"));
1179 1 : printf(_(" --install-missing install missing extensions\n"));
675 1180 1 : printf(_(" -?, --help show this help, then exit\n"));
1181 :
727 1182 1 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
1183 1 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
758 rhaas 1184 1 : }
1185 :
1186 : /*
1187 : * Print a progress report based on the global variables.
1188 : *
1189 : * Progress report is written at maximum once per second, unless the force
1190 : * parameter is set to true.
1191 : *
1192 : * If finished is set to true, this is the last progress report. The cursor
1193 : * is moved to the next line.
1194 : */
1195 : static void
1196 5880 : progress_report(uint64 relations_total, uint64 relations_checked,
1197 : uint64 relpages_total, uint64 relpages_checked,
1198 : const char *datname, bool force, bool finished)
1199 : {
1200 5880 : int percent_rel = 0;
1201 5880 : int percent_pages = 0;
1202 : char checked_rel[32];
1203 : char total_rel[32];
1204 : char checked_pages[32];
1205 : char total_pages[32];
1206 : pg_time_t now;
1207 :
1208 5880 : if (!opts.show_progress)
1209 5880 : return;
1210 :
758 rhaas 1211 UBC 0 : now = time(NULL);
1212 0 : if (now == last_progress_report && !force && !finished)
1213 0 : return; /* Max once per second */
1214 :
1215 0 : last_progress_report = now;
1216 0 : if (relations_total)
1217 0 : percent_rel = (int) (relations_checked * 100 / relations_total);
1218 0 : if (relpages_total)
1219 0 : percent_pages = (int) (relpages_checked * 100 / relpages_total);
1220 :
571 peter 1221 0 : snprintf(checked_rel, sizeof(checked_rel), UINT64_FORMAT, relations_checked);
1222 0 : snprintf(total_rel, sizeof(total_rel), UINT64_FORMAT, relations_total);
1223 0 : snprintf(checked_pages, sizeof(checked_pages), UINT64_FORMAT, relpages_checked);
1224 0 : snprintf(total_pages, sizeof(total_pages), UINT64_FORMAT, relpages_total);
1225 :
1226 : #define VERBOSE_DATNAME_LENGTH 35
758 rhaas 1227 0 : if (opts.verbose)
1228 : {
1229 0 : if (!datname)
1230 :
1231 : /*
1232 : * No datname given, so clear the status line (used for first and
1233 : * last call)
1234 : */
1235 0 : fprintf(stderr,
610 peter 1236 0 : _("%*s/%s relations (%d%%), %*s/%s pages (%d%%) %*s"),
758 rhaas 1237 0 : (int) strlen(total_rel),
1238 : checked_rel, total_rel, percent_rel,
1239 0 : (int) strlen(total_pages),
1240 : checked_pages, total_pages, percent_pages,
1241 : VERBOSE_DATNAME_LENGTH + 2, "");
1242 : else
1243 : {
1244 0 : bool truncate = (strlen(datname) > VERBOSE_DATNAME_LENGTH);
1245 :
1246 0 : fprintf(stderr,
610 peter 1247 0 : _("%*s/%s relations (%d%%), %*s/%s pages (%d%%) (%s%-*.*s)"),
758 rhaas 1248 0 : (int) strlen(total_rel),
1249 : checked_rel, total_rel, percent_rel,
1250 0 : (int) strlen(total_pages),
1251 : checked_pages, total_pages, percent_pages,
1252 : /* Prefix with "..." if we do leading truncation */
1253 : truncate ? "..." : "",
1254 : truncate ? VERBOSE_DATNAME_LENGTH - 3 : VERBOSE_DATNAME_LENGTH,
1255 : truncate ? VERBOSE_DATNAME_LENGTH - 3 : VERBOSE_DATNAME_LENGTH,
1256 : /* Truncate datname at beginning if it's too long */
1257 0 : truncate ? datname + strlen(datname) - VERBOSE_DATNAME_LENGTH + 3 : datname);
1258 : }
1259 : }
1260 : else
1261 0 : fprintf(stderr,
610 peter 1262 0 : _("%*s/%s relations (%d%%), %*s/%s pages (%d%%)"),
758 rhaas 1263 0 : (int) strlen(total_rel),
1264 : checked_rel, total_rel, percent_rel,
1265 0 : (int) strlen(total_pages),
1266 : checked_pages, total_pages, percent_pages);
1267 :
1268 : /*
1269 : * Stay on the same line if reporting to a terminal and we're not done
1270 : * yet.
1271 : */
1272 0 : if (!finished && isatty(fileno(stderr)))
1273 : {
1274 0 : fputc('\r', stderr);
1275 0 : progress_since_last_stderr = true;
1276 : }
1277 : else
1278 0 : fputc('\n', stderr);
1279 : }
1280 :
1281 : /*
1282 : * Extend the pattern info array to hold one additional initialized pattern
1283 : * info entry.
1284 : *
1285 : * Returns a pointer to the new entry.
1286 : */
1287 : static PatternInfo *
758 rhaas 1288 CBC 91 : extend_pattern_info_array(PatternInfoArray *pia)
1289 : {
1290 : PatternInfo *result;
1291 :
1292 91 : pia->len++;
1293 91 : pia->data = (PatternInfo *) pg_realloc(pia->data, pia->len * sizeof(PatternInfo));
1294 91 : result = &pia->data[pia->len - 1];
1295 91 : memset(result, 0, sizeof(*result));
1296 :
1297 91 : return result;
1298 : }
1299 :
1300 : /*
1301 : * append_database_pattern
1302 : *
1303 : * Adds the given pattern interpreted as a database name pattern.
1304 : *
1305 : * pia: the pattern info array to be appended
1306 : * pattern: the database name pattern
1307 : * encoding: client encoding for parsing the pattern
1308 : */
1309 : static void
1310 29 : append_database_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1311 : {
1312 : PQExpBufferData buf;
1313 : int dotcnt;
1314 29 : PatternInfo *info = extend_pattern_info_array(pia);
1315 :
1316 29 : initPQExpBuffer(&buf);
354 1317 29 : patternToSQLRegex(encoding, NULL, NULL, &buf, pattern, false, false,
1318 : &dotcnt);
1319 29 : if (dotcnt > 0)
1320 : {
1321 3 : pg_log_error("improper qualified name (too many dotted names): %s", pattern);
1322 3 : exit(2);
1323 : }
758 1324 26 : info->pattern = pattern;
1325 26 : info->db_regex = pstrdup(buf.data);
1326 :
1327 26 : termPQExpBuffer(&buf);
1328 26 : }
1329 :
1330 : /*
1331 : * append_schema_pattern
1332 : *
1333 : * Adds the given pattern interpreted as a schema name pattern.
1334 : *
1335 : * pia: the pattern info array to be appended
1336 : * pattern: the schema name pattern
1337 : * encoding: client encoding for parsing the pattern
1338 : */
1339 : static void
1340 26 : append_schema_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1341 : {
1342 : PQExpBufferData dbbuf;
1343 : PQExpBufferData nspbuf;
1344 : int dotcnt;
1345 26 : PatternInfo *info = extend_pattern_info_array(pia);
1346 :
1347 26 : initPQExpBuffer(&dbbuf);
1348 26 : initPQExpBuffer(&nspbuf);
1349 :
354 1350 26 : patternToSQLRegex(encoding, NULL, &dbbuf, &nspbuf, pattern, false, false,
1351 : &dotcnt);
1352 26 : if (dotcnt > 1)
1353 : {
1354 3 : pg_log_error("improper qualified name (too many dotted names): %s", pattern);
1355 3 : exit(2);
1356 : }
758 1357 23 : info->pattern = pattern;
1358 23 : if (dbbuf.data[0])
1359 : {
758 rhaas 1360 UBC 0 : opts.dbpattern = true;
1361 0 : info->db_regex = pstrdup(dbbuf.data);
1362 : }
758 rhaas 1363 CBC 23 : if (nspbuf.data[0])
1364 23 : info->nsp_regex = pstrdup(nspbuf.data);
1365 :
1366 23 : termPQExpBuffer(&dbbuf);
1367 23 : termPQExpBuffer(&nspbuf);
1368 23 : }
1369 :
1370 : /*
1371 : * append_relation_pattern_helper
1372 : *
1373 : * Adds to a list the given pattern interpreted as a relation pattern.
1374 : *
1375 : * pia: the pattern info array to be appended
1376 : * pattern: the relation name pattern
1377 : * encoding: client encoding for parsing the pattern
1378 : * heap_only: whether the pattern should only be matched against heap tables
1379 : * btree_only: whether the pattern should only be matched against btree indexes
1380 : */
1381 : static void
1382 36 : append_relation_pattern_helper(PatternInfoArray *pia, const char *pattern,
1383 : int encoding, bool heap_only, bool btree_only)
1384 : {
1385 : PQExpBufferData dbbuf;
1386 : PQExpBufferData nspbuf;
1387 : PQExpBufferData relbuf;
1388 : int dotcnt;
1389 36 : PatternInfo *info = extend_pattern_info_array(pia);
1390 :
1391 36 : initPQExpBuffer(&dbbuf);
1392 36 : initPQExpBuffer(&nspbuf);
1393 36 : initPQExpBuffer(&relbuf);
1394 :
354 1395 36 : patternToSQLRegex(encoding, &dbbuf, &nspbuf, &relbuf, pattern, false,
1396 : false, &dotcnt);
1397 36 : if (dotcnt > 2)
1398 : {
1399 3 : pg_log_error("improper relation name (too many dotted names): %s", pattern);
1400 3 : exit(2);
1401 : }
758 1402 33 : info->pattern = pattern;
1403 33 : if (dbbuf.data[0])
1404 : {
1405 13 : opts.dbpattern = true;
1406 13 : info->db_regex = pstrdup(dbbuf.data);
1407 : }
1408 33 : if (nspbuf.data[0])
1409 19 : info->nsp_regex = pstrdup(nspbuf.data);
1410 33 : if (relbuf.data[0])
1411 33 : info->rel_regex = pstrdup(relbuf.data);
1412 :
1413 33 : termPQExpBuffer(&dbbuf);
1414 33 : termPQExpBuffer(&nspbuf);
1415 33 : termPQExpBuffer(&relbuf);
1416 :
1417 33 : info->heap_only = heap_only;
1418 33 : info->btree_only = btree_only;
1419 33 : }
1420 :
1421 : /*
1422 : * append_relation_pattern
1423 : *
1424 : * Adds the given pattern interpreted as a relation pattern, to be matched
1425 : * against both heap tables and btree indexes.
1426 : *
1427 : * pia: the pattern info array to be appended
1428 : * pattern: the relation name pattern
1429 : * encoding: client encoding for parsing the pattern
1430 : */
1431 : static void
1432 7 : append_relation_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1433 : {
1434 7 : append_relation_pattern_helper(pia, pattern, encoding, false, false);
1435 7 : }
1436 :
1437 : /*
1438 : * append_heap_pattern
1439 : *
1440 : * Adds the given pattern interpreted as a relation pattern, to be matched only
1441 : * against heap tables.
1442 : *
1443 : * pia: the pattern info array to be appended
1444 : * pattern: the relation name pattern
1445 : * encoding: client encoding for parsing the pattern
1446 : */
1447 : static void
1448 18 : append_heap_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1449 : {
1450 18 : append_relation_pattern_helper(pia, pattern, encoding, true, false);
1451 15 : }
1452 :
1453 : /*
1454 : * append_btree_pattern
1455 : *
1456 : * Adds the given pattern interpreted as a relation pattern, to be matched only
1457 : * against btree indexes.
1458 : *
1459 : * pia: the pattern info array to be appended
1460 : * pattern: the relation name pattern
1461 : * encoding: client encoding for parsing the pattern
1462 : */
1463 : static void
1464 11 : append_btree_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1465 : {
1466 11 : append_relation_pattern_helper(pia, pattern, encoding, false, true);
1467 11 : }
1468 :
1469 : /*
1470 : * append_db_pattern_cte
1471 : *
1472 : * Appends to the buffer the body of a Common Table Expression (CTE) containing
1473 : * the database portions filtered from the list of patterns expressed as two
1474 : * columns:
1475 : *
1476 : * pattern_id: the index of this pattern in pia->data[]
1477 : * rgx: the database regular expression parsed from the pattern
1478 : *
1479 : * Patterns without a database portion are skipped. Patterns with more than
1480 : * just a database portion are optionally skipped, depending on argument
1481 : * 'inclusive'.
1482 : *
1483 : * buf: the buffer to be appended
1484 : * pia: the array of patterns to be inserted into the CTE
1485 : * conn: the database connection
1486 : * inclusive: whether to include patterns with schema and/or relation parts
1487 : *
1488 : * Returns whether any database patterns were appended.
1489 : */
1490 : static bool
1491 54 : append_db_pattern_cte(PQExpBuffer buf, const PatternInfoArray *pia,
1492 : PGconn *conn, bool inclusive)
1493 : {
1494 : int pattern_id;
1495 : const char *comma;
1496 : bool have_values;
1497 :
1498 54 : comma = "";
1499 54 : have_values = false;
1500 129 : for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1501 : {
1502 75 : PatternInfo *info = &pia->data[pattern_id];
1503 :
1504 75 : if (info->db_regex != NULL &&
758 rhaas 1505 UBC 0 : (inclusive || (info->nsp_regex == NULL && info->rel_regex == NULL)))
1506 : {
758 rhaas 1507 CBC 39 : if (!have_values)
1508 14 : appendPQExpBufferStr(buf, "\nVALUES");
1509 39 : have_values = true;
1510 39 : appendPQExpBuffer(buf, "%s\n(%d, ", comma, pattern_id);
1511 39 : appendStringLiteralConn(buf, info->db_regex, conn);
215 drowley 1512 GNC 39 : appendPQExpBufferChar(buf, ')');
758 rhaas 1513 CBC 39 : comma = ",";
1514 : }
1515 : }
1516 :
1517 54 : if (!have_values)
1518 40 : appendPQExpBufferStr(buf, "\nSELECT NULL, NULL, NULL WHERE false");
1519 :
1520 54 : return have_values;
1521 : }
1522 :
1523 : /*
1524 : * compile_database_list
1525 : *
1526 : * If any database patterns exist, or if --all was given, compiles a distinct
1527 : * list of databases to check using a SQL query based on the patterns plus the
1528 : * literal initial database name, if given. If no database patterns exist and
1529 : * --all was not given, the query is not necessary, and only the initial
1530 : * database name (if any) is added to the list.
1531 : *
1532 : * conn: connection to the initial database
1533 : * databases: the list onto which databases should be appended
1534 : * initial_dbname: an optional extra database name to include in the list
1535 : */
1536 : static void
1537 36 : compile_database_list(PGconn *conn, SimplePtrList *databases,
1538 : const char *initial_dbname)
1539 : {
1540 : PGresult *res;
1541 : PQExpBufferData sql;
1542 : int ntups;
1543 : int i;
1544 : bool fatal;
1545 :
1546 36 : if (initial_dbname)
1547 : {
1548 18 : DatabaseInfo *dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1549 :
1550 : /* This database is included. Add to list */
1551 18 : if (opts.verbose)
696 peter 1552 UBC 0 : pg_log_info("including database \"%s\"", initial_dbname);
1553 :
758 rhaas 1554 CBC 18 : dat->datname = pstrdup(initial_dbname);
1555 18 : simple_ptr_list_append(databases, dat);
1556 : }
1557 :
1558 36 : initPQExpBuffer(&sql);
1559 :
1560 : /* Append the include patterns CTE. */
1561 36 : appendPQExpBufferStr(&sql, "WITH include_raw (pattern_id, rgx) AS (");
1562 36 : if (!append_db_pattern_cte(&sql, &opts.include, conn, true) &&
1563 22 : !opts.alldb)
1564 : {
1565 : /*
1566 : * None of the inclusion patterns (if any) contain database portions,
1567 : * so there is no need to query the database to resolve database
1568 : * patterns.
1569 : *
1570 : * Since we're also not operating under --all, we don't need to query
1571 : * the exhaustive list of connectable databases, either.
1572 : */
1573 18 : termPQExpBuffer(&sql);
1574 18 : return;
1575 : }
1576 :
1577 : /* Append the exclude patterns CTE. */
1578 18 : appendPQExpBufferStr(&sql, "),\nexclude_raw (pattern_id, rgx) AS (");
1579 18 : append_db_pattern_cte(&sql, &opts.exclude, conn, false);
1580 18 : appendPQExpBufferStr(&sql, "),");
1581 :
1582 : /*
1583 : * Append the database CTE, which includes whether each database is
1584 : * connectable and also joins against exclude_raw to determine whether
1585 : * each database is excluded.
1586 : */
1587 18 : appendPQExpBufferStr(&sql,
1588 : "\ndatabase (datname) AS ("
1589 : "\nSELECT d.datname "
1590 : "FROM pg_catalog.pg_database d "
1591 : "LEFT OUTER JOIN exclude_raw e "
1592 : "ON d.datname ~ e.rgx "
1593 : "\nWHERE d.datallowconn "
1594 : "AND e.pattern_id IS NULL"
1595 : "),"
1596 :
1597 : /*
1598 : * Append the include_pat CTE, which joins the include_raw CTE against the
1599 : * databases CTE to determine if all the inclusion patterns had matches,
1600 : * and whether each matched pattern had the misfortune of only matching
1601 : * excluded or unconnectable databases.
1602 : */
1603 : "\ninclude_pat (pattern_id, checkable) AS ("
1604 : "\nSELECT i.pattern_id, "
1605 : "COUNT(*) FILTER ("
1606 : "WHERE d IS NOT NULL"
1607 : ") AS checkable"
1608 : "\nFROM include_raw i "
1609 : "LEFT OUTER JOIN database d "
1610 : "ON d.datname ~ i.rgx"
1611 : "\nGROUP BY i.pattern_id"
1612 : "),"
1613 :
1614 : /*
1615 : * Append the filtered_databases CTE, which selects from the database CTE
1616 : * optionally joined against the include_raw CTE to only select databases
1617 : * that match an inclusion pattern. This appears to duplicate what the
1618 : * include_pat CTE already did above, but here we want only databases, and
1619 : * there we wanted patterns.
1620 : */
1621 : "\nfiltered_databases (datname) AS ("
1622 : "\nSELECT DISTINCT d.datname "
1623 : "FROM database d");
1624 18 : if (!opts.alldb)
1625 14 : appendPQExpBufferStr(&sql,
1626 : " INNER JOIN include_raw i "
1627 : "ON d.datname ~ i.rgx");
1628 18 : appendPQExpBufferStr(&sql,
1629 : ")"
1630 :
1631 : /*
1632 : * Select the checkable databases and the unmatched inclusion patterns.
1633 : */
1634 : "\nSELECT pattern_id, datname FROM ("
1635 : "\nSELECT pattern_id, NULL::TEXT AS datname "
1636 : "FROM include_pat "
1637 : "WHERE checkable = 0 "
1638 : "UNION ALL"
1639 : "\nSELECT NULL, datname "
1640 : "FROM filtered_databases"
1641 : ") AS combined_records"
1642 : "\nORDER BY pattern_id NULLS LAST, datname");
1643 :
1644 18 : res = executeQuery(conn, sql.data, opts.echo);
1645 18 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
1646 : {
758 rhaas 1647 UBC 0 : pg_log_error("query failed: %s", PQerrorMessage(conn));
366 tgl 1648 0 : pg_log_error_detail("Query was: %s", sql.data);
758 rhaas 1649 0 : disconnectDatabase(conn);
1650 0 : exit(1);
1651 : }
758 rhaas 1652 CBC 18 : termPQExpBuffer(&sql);
1653 :
1654 18 : ntups = PQntuples(res);
1655 65 : for (fatal = false, i = 0; i < ntups; i++)
1656 : {
1657 47 : int pattern_id = -1;
1658 47 : const char *datname = NULL;
1659 :
1660 47 : if (!PQgetisnull(res, i, 0))
1661 11 : pattern_id = atoi(PQgetvalue(res, i, 0));
1662 47 : if (!PQgetisnull(res, i, 1))
1663 36 : datname = PQgetvalue(res, i, 1);
1664 :
1665 47 : if (pattern_id >= 0)
1666 : {
1667 : /*
1668 : * Current record pertains to an inclusion pattern that matched no
1669 : * checkable databases.
1670 : */
1671 11 : fatal = opts.strict_names;
1672 11 : if (pattern_id >= opts.include.len)
366 tgl 1673 UBC 0 : pg_fatal("internal error: received unexpected database pattern_id %d",
1674 : pattern_id);
758 rhaas 1675 CBC 11 : log_no_match("no connectable databases to check matching \"%s\"",
1676 : opts.include.data[pattern_id].pattern);
1677 : }
1678 : else
1679 : {
1680 : DatabaseInfo *dat;
1681 :
1682 : /* Current record pertains to a database */
1683 36 : Assert(datname != NULL);
1684 :
1685 : /* Avoid entering a duplicate entry matching the initial_dbname */
1686 36 : if (initial_dbname != NULL && strcmp(initial_dbname, datname) == 0)
758 rhaas 1687 UBC 0 : continue;
1688 :
1689 : /* This database is included. Add to list */
758 rhaas 1690 CBC 36 : if (opts.verbose)
696 peter 1691 UBC 0 : pg_log_info("including database \"%s\"", datname);
1692 :
758 rhaas 1693 CBC 36 : dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1694 36 : dat->datname = pstrdup(datname);
1695 36 : simple_ptr_list_append(databases, dat);
1696 : }
1697 : }
1698 18 : PQclear(res);
1699 :
1700 18 : if (fatal)
1701 : {
1702 5 : if (conn != NULL)
1703 5 : disconnectDatabase(conn);
1704 5 : exit(1);
1705 : }
1706 : }
1707 :
1708 : /*
1709 : * append_rel_pattern_raw_cte
1710 : *
1711 : * Appends to the buffer the body of a Common Table Expression (CTE) containing
1712 : * the given patterns as six columns:
1713 : *
1714 : * pattern_id: the index of this pattern in pia->data[]
1715 : * db_regex: the database regexp parsed from the pattern, or NULL if the
1716 : * pattern had no database part
1717 : * nsp_regex: the namespace regexp parsed from the pattern, or NULL if the
1718 : * pattern had no namespace part
1719 : * rel_regex: the relname regexp parsed from the pattern, or NULL if the
1720 : * pattern had no relname part
1721 : * heap_only: true if the pattern applies only to heap tables (not indexes)
1722 : * btree_only: true if the pattern applies only to btree indexes (not tables)
1723 : *
1724 : * buf: the buffer to be appended
1725 : * patterns: the array of patterns to be inserted into the CTE
1726 : * conn: the database connection
1727 : */
1728 : static void
1729 26 : append_rel_pattern_raw_cte(PQExpBuffer buf, const PatternInfoArray *pia,
1730 : PGconn *conn)
1731 : {
1732 : int pattern_id;
1733 : const char *comma;
1734 : bool have_values;
1735 :
1736 26 : comma = "";
1737 26 : have_values = false;
1738 88 : for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1739 : {
1740 62 : PatternInfo *info = &pia->data[pattern_id];
1741 :
1742 62 : if (!have_values)
1743 26 : appendPQExpBufferStr(buf, "\nVALUES");
1744 62 : have_values = true;
1745 62 : appendPQExpBuffer(buf, "%s\n(%d::INTEGER, ", comma, pattern_id);
1746 62 : if (info->db_regex == NULL)
1747 46 : appendPQExpBufferStr(buf, "NULL");
1748 : else
1749 16 : appendStringLiteralConn(buf, info->db_regex, conn);
1750 62 : appendPQExpBufferStr(buf, "::TEXT, ");
1751 62 : if (info->nsp_regex == NULL)
1752 21 : appendPQExpBufferStr(buf, "NULL");
1753 : else
1754 41 : appendStringLiteralConn(buf, info->nsp_regex, conn);
1755 62 : appendPQExpBufferStr(buf, "::TEXT, ");
1756 62 : if (info->rel_regex == NULL)
1757 29 : appendPQExpBufferStr(buf, "NULL");
1758 : else
1759 33 : appendStringLiteralConn(buf, info->rel_regex, conn);
1760 62 : if (info->heap_only)
1761 13 : appendPQExpBufferStr(buf, "::TEXT, true::BOOLEAN");
1762 : else
1763 49 : appendPQExpBufferStr(buf, "::TEXT, false::BOOLEAN");
1764 62 : if (info->btree_only)
1765 13 : appendPQExpBufferStr(buf, ", true::BOOLEAN");
1766 : else
1767 49 : appendPQExpBufferStr(buf, ", false::BOOLEAN");
215 drowley 1768 GNC 62 : appendPQExpBufferChar(buf, ')');
758 rhaas 1769 CBC 62 : comma = ",";
1770 : }
1771 :
1772 26 : if (!have_values)
758 rhaas 1773 UBC 0 : appendPQExpBufferStr(buf,
1774 : "\nSELECT NULL::INTEGER, NULL::TEXT, NULL::TEXT, "
1775 : "NULL::TEXT, NULL::BOOLEAN, NULL::BOOLEAN "
1776 : "WHERE false");
758 rhaas 1777 CBC 26 : }
1778 :
1779 : /*
1780 : * append_rel_pattern_filtered_cte
1781 : *
1782 : * Appends to the buffer a Common Table Expression (CTE) which selects
1783 : * all patterns from the named raw CTE, filtered by database. All patterns
1784 : * which have no database portion or whose database portion matches our
1785 : * connection's database name are selected, with other patterns excluded.
1786 : *
1787 : * The basic idea here is that if we're connected to database "foo" and we have
1788 : * patterns "foo.bar.baz", "alpha.beta" and "one.two.three", we only want to
1789 : * use the first two while processing relations in this database, as the third
1790 : * one is not relevant.
1791 : *
1792 : * buf: the buffer to be appended
1793 : * raw: the name of the CTE to select from
1794 : * filtered: the name of the CTE to create
1795 : * conn: the database connection
1796 : */
1797 : static void
1798 26 : append_rel_pattern_filtered_cte(PQExpBuffer buf, const char *raw,
1799 : const char *filtered, PGconn *conn)
1800 : {
1801 26 : appendPQExpBuffer(buf,
1802 : "\n%s (pattern_id, nsp_regex, rel_regex, heap_only, btree_only) AS ("
1803 : "\nSELECT pattern_id, nsp_regex, rel_regex, heap_only, btree_only "
1804 : "FROM %s r"
1805 : "\nWHERE (r.db_regex IS NULL "
1806 : "OR ",
1807 : filtered, raw);
1808 26 : appendStringLiteralConn(buf, PQdb(conn), conn);
1809 26 : appendPQExpBufferStr(buf, " ~ r.db_regex)");
1810 26 : appendPQExpBufferStr(buf,
1811 : " AND (r.nsp_regex IS NOT NULL"
1812 : " OR r.rel_regex IS NOT NULL)"
1813 : "),");
1814 26 : }
1815 :
1816 : /*
1817 : * compile_relation_list_one_db
1818 : *
1819 : * Compiles a list of relations to check within the currently connected
1820 : * database based on the user supplied options, sorted by descending size,
1821 : * and appends them to the given list of relations.
1822 : *
1823 : * The cells of the constructed list contain all information about the relation
1824 : * necessary to connect to the database and check the object, including which
1825 : * database to connect to, where contrib/amcheck is installed, and the Oid and
1826 : * type of object (heap table vs. btree index). Rather than duplicating the
1827 : * database details per relation, the relation structs use references to the
1828 : * same database object, provided by the caller.
1829 : *
1830 : * conn: connection to this next database, which should be the same as in 'dat'
1831 : * relations: list onto which the relations information should be appended
1832 : * dat: the database info struct for use by each relation
1833 : * pagecount: gets incremented by the number of blocks to check in all
1834 : * relations added
1835 : */
1836 : static void
1837 38 : compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations,
1838 : const DatabaseInfo *dat,
1839 : uint64 *pagecount)
1840 : {
1841 : PGresult *res;
1842 : PQExpBufferData sql;
1843 : int ntups;
1844 : int i;
1845 :
1846 38 : initPQExpBuffer(&sql);
1847 38 : appendPQExpBufferStr(&sql, "WITH");
1848 :
1849 : /* Append CTEs for the relation inclusion patterns, if any */
1850 38 : if (!opts.allrel)
1851 : {
1852 19 : appendPQExpBufferStr(&sql,
1853 : " include_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1854 19 : append_rel_pattern_raw_cte(&sql, &opts.include, conn);
1855 19 : appendPQExpBufferStr(&sql, "\n),");
1856 19 : append_rel_pattern_filtered_cte(&sql, "include_raw", "include_pat", conn);
1857 : }
1858 :
1859 : /* Append CTEs for the relation exclusion patterns, if any */
1860 38 : if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1861 : {
1862 7 : appendPQExpBufferStr(&sql,
1863 : " exclude_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1864 7 : append_rel_pattern_raw_cte(&sql, &opts.exclude, conn);
1865 7 : appendPQExpBufferStr(&sql, "\n),");
1866 7 : append_rel_pattern_filtered_cte(&sql, "exclude_raw", "exclude_pat", conn);
1867 : }
1868 :
1869 : /* Append the relation CTE. */
1870 38 : appendPQExpBufferStr(&sql,
1871 : " relation (pattern_id, oid, nspname, relname, reltoastrelid, relpages, is_heap, is_btree) AS ("
1872 : "\nSELECT DISTINCT ON (c.oid");
1873 38 : if (!opts.allrel)
1874 19 : appendPQExpBufferStr(&sql, ", ip.pattern_id) ip.pattern_id,");
1875 : else
1876 19 : appendPQExpBufferStr(&sql, ") NULL::INTEGER AS pattern_id,");
1877 38 : appendPQExpBuffer(&sql,
1878 : "\nc.oid, n.nspname, c.relname, c.reltoastrelid, c.relpages, "
1879 : "c.relam = %u AS is_heap, "
1880 : "c.relam = %u AS is_btree"
1881 : "\nFROM pg_catalog.pg_class c "
1882 : "INNER JOIN pg_catalog.pg_namespace n "
1883 : "ON c.relnamespace = n.oid",
1884 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
1885 38 : if (!opts.allrel)
1886 19 : appendPQExpBuffer(&sql,
1887 : "\nINNER JOIN include_pat ip"
1888 : "\nON (n.nspname ~ ip.nsp_regex OR ip.nsp_regex IS NULL)"
1889 : "\nAND (c.relname ~ ip.rel_regex OR ip.rel_regex IS NULL)"
1890 : "\nAND (c.relam = %u OR NOT ip.heap_only)"
1891 : "\nAND (c.relam = %u OR NOT ip.btree_only)",
1892 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
1893 38 : if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1894 7 : appendPQExpBuffer(&sql,
1895 : "\nLEFT OUTER JOIN exclude_pat ep"
1896 : "\nON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
1897 : "\nAND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
1898 : "\nAND (c.relam = %u OR NOT ep.heap_only OR ep.rel_regex IS NULL)"
1899 : "\nAND (c.relam = %u OR NOT ep.btree_only OR ep.rel_regex IS NULL)",
1900 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
1901 :
1902 : /*
1903 : * Exclude temporary tables and indexes, which must necessarily belong to
1904 : * other sessions. (We don't create any ourselves.) We must ultimately
1905 : * exclude indexes marked invalid or not ready, but we delay that decision
1906 : * until firing off the amcheck command, as the state of an index may
1907 : * change by then.
1908 : */
543 pg 1909 38 : appendPQExpBufferStr(&sql, "\nWHERE c.relpersistence != 't'");
758 rhaas 1910 38 : if (opts.excludetbl || opts.excludeidx || opts.excludensp)
543 pg 1911 7 : appendPQExpBufferStr(&sql, "\nAND ep.pattern_id IS NULL");
1912 :
1913 : /*
1914 : * We need to be careful not to break the --no-dependent-toast and
1915 : * --no-dependent-indexes options. By default, the btree indexes, toast
1916 : * tables, and toast table btree indexes associated with primary heap
1917 : * tables are included, using their own CTEs below. We implement the
1918 : * --exclude-* options by not creating those CTEs, but that's no use if
1919 : * we've already selected the toast and indexes here. On the other hand,
1920 : * we want inclusion patterns that match indexes or toast tables to be
1921 : * honored. So, if inclusion patterns were given, we want to select all
1922 : * tables, toast tables, or indexes that match the patterns. But if no
1923 : * inclusion patterns were given, and we're simply matching all relations,
1924 : * then we only want to match the primary tables here.
1925 : */
758 rhaas 1926 38 : if (opts.allrel)
1927 19 : appendPQExpBuffer(&sql,
1928 : " AND c.relam = %u "
1929 : "AND c.relkind IN ('r', 'S', 'm', 't') "
1930 : "AND c.relnamespace != %u",
1931 : HEAP_TABLE_AM_OID, PG_TOAST_NAMESPACE);
1932 : else
1933 19 : appendPQExpBuffer(&sql,
1934 : " AND c.relam IN (%u, %u)"
1935 : "AND c.relkind IN ('r', 'S', 'm', 't', 'i') "
1936 : "AND ((c.relam = %u AND c.relkind IN ('r', 'S', 'm', 't')) OR "
1937 : "(c.relam = %u AND c.relkind = 'i'))",
1938 : HEAP_TABLE_AM_OID, BTREE_AM_OID,
1939 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
1940 :
1941 38 : appendPQExpBufferStr(&sql,
1942 : "\nORDER BY c.oid)");
1943 :
1944 38 : if (!opts.no_toast_expansion)
1945 : {
1946 : /*
1947 : * Include a CTE for toast tables associated with primary heap tables
1948 : * selected above, filtering by exclusion patterns (if any) that match
1949 : * toast table names.
1950 : */
1951 37 : appendPQExpBufferStr(&sql,
1952 : ", toast (oid, nspname, relname, relpages) AS ("
1953 : "\nSELECT t.oid, 'pg_toast', t.relname, t.relpages"
1954 : "\nFROM pg_catalog.pg_class t "
1955 : "INNER JOIN relation r "
1956 : "ON r.reltoastrelid = t.oid");
1957 37 : if (opts.excludetbl || opts.excludensp)
1958 6 : appendPQExpBufferStr(&sql,
1959 : "\nLEFT OUTER JOIN exclude_pat ep"
1960 : "\nON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
1961 : "\nAND (t.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
1962 : "\nAND ep.heap_only"
1963 : "\nWHERE ep.pattern_id IS NULL"
1964 : "\nAND t.relpersistence != 't'");
1965 37 : appendPQExpBufferStr(&sql,
1966 : "\n)");
1967 : }
1968 38 : if (!opts.no_btree_expansion)
1969 : {
1970 : /*
1971 : * Include a CTE for btree indexes associated with primary heap tables
1972 : * selected above, filtering by exclusion patterns (if any) that match
1973 : * btree index names.
1974 : */
215 drowley 1975 GNC 34 : appendPQExpBufferStr(&sql,
1976 : ", index (oid, nspname, relname, relpages) AS ("
1977 : "\nSELECT c.oid, r.nspname, c.relname, c.relpages "
1978 : "FROM relation r"
1979 : "\nINNER JOIN pg_catalog.pg_index i "
1980 : "ON r.oid = i.indrelid "
1981 : "INNER JOIN pg_catalog.pg_class c "
1982 : "ON i.indexrelid = c.oid "
1983 : "AND c.relpersistence != 't'");
758 rhaas 1984 CBC 34 : if (opts.excludeidx || opts.excludensp)
1985 6 : appendPQExpBufferStr(&sql,
1986 : "\nINNER JOIN pg_catalog.pg_namespace n "
1987 : "ON c.relnamespace = n.oid"
1988 : "\nLEFT OUTER JOIN exclude_pat ep "
1989 : "ON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
1990 : "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
1991 : "AND ep.btree_only"
1992 : "\nWHERE ep.pattern_id IS NULL");
1993 : else
1994 28 : appendPQExpBufferStr(&sql,
1995 : "\nWHERE true");
1996 34 : appendPQExpBuffer(&sql,
1997 : " AND c.relam = %u "
1998 : "AND c.relkind = 'i'",
1999 : BTREE_AM_OID);
2000 34 : if (opts.no_toast_expansion)
2001 1 : appendPQExpBuffer(&sql,
2002 : " AND c.relnamespace != %u",
2003 : PG_TOAST_NAMESPACE);
2004 34 : appendPQExpBufferStr(&sql, "\n)");
2005 : }
2006 :
2007 38 : if (!opts.no_toast_expansion && !opts.no_btree_expansion)
2008 : {
2009 : /*
2010 : * Include a CTE for btree indexes associated with toast tables of
2011 : * primary heap tables selected above, filtering by exclusion patterns
2012 : * (if any) that match the toast index names.
2013 : */
215 drowley 2014 GNC 33 : appendPQExpBufferStr(&sql,
2015 : ", toast_index (oid, nspname, relname, relpages) AS ("
2016 : "\nSELECT c.oid, 'pg_toast', c.relname, c.relpages "
2017 : "FROM toast t "
2018 : "INNER JOIN pg_catalog.pg_index i "
2019 : "ON t.oid = i.indrelid"
2020 : "\nINNER JOIN pg_catalog.pg_class c "
2021 : "ON i.indexrelid = c.oid "
2022 : "AND c.relpersistence != 't'");
758 rhaas 2023 CBC 33 : if (opts.excludeidx)
2024 1 : appendPQExpBufferStr(&sql,
2025 : "\nLEFT OUTER JOIN exclude_pat ep "
2026 : "ON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
2027 : "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
2028 : "AND ep.btree_only "
2029 : "WHERE ep.pattern_id IS NULL");
2030 : else
2031 32 : appendPQExpBufferStr(&sql,
2032 : "\nWHERE true");
2033 33 : appendPQExpBuffer(&sql,
2034 : " AND c.relam = %u"
2035 : " AND c.relkind = 'i')",
2036 : BTREE_AM_OID);
2037 : }
2038 :
2039 : /*
2040 : * Roll-up distinct rows from CTEs.
2041 : *
2042 : * Relations that match more than one pattern may occur more than once in
2043 : * the list, and indexes and toast for primary relations may also have
2044 : * matched in their own right, so we rely on UNION to deduplicate the
2045 : * list.
2046 : */
215 drowley 2047 GNC 38 : appendPQExpBufferStr(&sql,
2048 : "\nSELECT pattern_id, is_heap, is_btree, oid, nspname, relname, relpages "
2049 : "FROM (");
758 rhaas 2050 CBC 38 : appendPQExpBufferStr(&sql,
2051 : /* Inclusion patterns that failed to match */
2052 : "\nSELECT pattern_id, is_heap, is_btree, "
2053 : "NULL::OID AS oid, "
2054 : "NULL::TEXT AS nspname, "
2055 : "NULL::TEXT AS relname, "
2056 : "NULL::INTEGER AS relpages"
2057 : "\nFROM relation "
2058 : "WHERE pattern_id IS NOT NULL "
2059 : "UNION"
2060 : /* Primary relations */
2061 : "\nSELECT NULL::INTEGER AS pattern_id, "
2062 : "is_heap, is_btree, oid, nspname, relname, relpages "
2063 : "FROM relation");
2064 38 : if (!opts.no_toast_expansion)
2065 37 : appendPQExpBufferStr(&sql,
2066 : " UNION"
2067 : /* Toast tables for primary relations */
2068 : "\nSELECT NULL::INTEGER AS pattern_id, TRUE AS is_heap, "
2069 : "FALSE AS is_btree, oid, nspname, relname, relpages "
2070 : "FROM toast");
2071 38 : if (!opts.no_btree_expansion)
2072 34 : appendPQExpBufferStr(&sql,
2073 : " UNION"
2074 : /* Indexes for primary relations */
2075 : "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2076 : "TRUE AS is_btree, oid, nspname, relname, relpages "
2077 : "FROM index");
2078 38 : if (!opts.no_toast_expansion && !opts.no_btree_expansion)
2079 33 : appendPQExpBufferStr(&sql,
2080 : " UNION"
2081 : /* Indexes for toast relations */
2082 : "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2083 : "TRUE AS is_btree, oid, nspname, relname, relpages "
2084 : "FROM toast_index");
2085 38 : appendPQExpBufferStr(&sql,
2086 : "\n) AS combined_records "
2087 : "ORDER BY relpages DESC NULLS FIRST, oid");
2088 :
2089 38 : res = executeQuery(conn, sql.data, opts.echo);
2090 38 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
2091 : {
758 rhaas 2092 UBC 0 : pg_log_error("query failed: %s", PQerrorMessage(conn));
366 tgl 2093 0 : pg_log_error_detail("Query was: %s", sql.data);
758 rhaas 2094 0 : disconnectDatabase(conn);
2095 0 : exit(1);
2096 : }
758 rhaas 2097 CBC 38 : termPQExpBuffer(&sql);
2098 :
2099 38 : ntups = PQntuples(res);
2100 5899 : for (i = 0; i < ntups; i++)
2101 : {
2102 5861 : int pattern_id = -1;
2103 5861 : bool is_heap = false;
2104 5861 : bool is_btree PG_USED_FOR_ASSERTS_ONLY = false;
2105 5861 : Oid oid = InvalidOid;
2106 5861 : const char *nspname = NULL;
2107 5861 : const char *relname = NULL;
2108 5861 : int relpages = 0;
2109 :
2110 5861 : if (!PQgetisnull(res, i, 0))
2111 33 : pattern_id = atoi(PQgetvalue(res, i, 0));
2112 5861 : if (!PQgetisnull(res, i, 1))
2113 5861 : is_heap = (PQgetvalue(res, i, 1)[0] == 't');
2114 5861 : if (!PQgetisnull(res, i, 2))
2115 5861 : is_btree = (PQgetvalue(res, i, 2)[0] == 't');
2116 5861 : if (!PQgetisnull(res, i, 3))
2117 5828 : oid = atooid(PQgetvalue(res, i, 3));
2118 5861 : if (!PQgetisnull(res, i, 4))
2119 5828 : nspname = PQgetvalue(res, i, 4);
2120 5861 : if (!PQgetisnull(res, i, 5))
2121 5828 : relname = PQgetvalue(res, i, 5);
2122 5861 : if (!PQgetisnull(res, i, 6))
2123 5828 : relpages = atoi(PQgetvalue(res, i, 6));
2124 :
2125 5861 : if (pattern_id >= 0)
2126 : {
2127 : /*
2128 : * Current record pertains to an inclusion pattern. Record that
2129 : * it matched.
2130 : */
2131 :
2132 33 : if (pattern_id >= opts.include.len)
366 tgl 2133 UBC 0 : pg_fatal("internal error: received unexpected relation pattern_id %d",
2134 : pattern_id);
2135 :
758 rhaas 2136 CBC 33 : opts.include.data[pattern_id].matched = true;
2137 : }
2138 : else
2139 : {
2140 : /* Current record pertains to a relation */
2141 :
2142 5828 : RelationInfo *rel = (RelationInfo *) pg_malloc0(sizeof(RelationInfo));
2143 :
2144 5828 : Assert(OidIsValid(oid));
2145 5828 : Assert((is_heap && !is_btree) || (is_btree && !is_heap));
2146 :
2147 5828 : rel->datinfo = dat;
2148 5828 : rel->reloid = oid;
2149 5828 : rel->is_heap = is_heap;
2150 5828 : rel->nspname = pstrdup(nspname);
2151 5828 : rel->relname = pstrdup(relname);
2152 5828 : rel->relpages = relpages;
2153 5828 : rel->blocks_to_check = relpages;
2154 5828 : if (is_heap && (opts.startblock >= 0 || opts.endblock >= 0))
2155 : {
2156 : /*
2157 : * We apply --startblock and --endblock to heap tables, but
2158 : * not btree indexes, and for progress purposes we need to
2159 : * track how many blocks we expect to check.
2160 : */
758 rhaas 2161 UBC 0 : if (opts.endblock >= 0 && rel->blocks_to_check > opts.endblock)
2162 0 : rel->blocks_to_check = opts.endblock + 1;
2163 0 : if (opts.startblock >= 0)
2164 : {
2165 0 : if (rel->blocks_to_check > opts.startblock)
2166 0 : rel->blocks_to_check -= opts.startblock;
2167 : else
2168 0 : rel->blocks_to_check = 0;
2169 : }
2170 : }
758 rhaas 2171 CBC 5828 : *pagecount += rel->blocks_to_check;
2172 :
2173 5828 : simple_ptr_list_append(relations, rel);
2174 : }
2175 : }
2176 38 : PQclear(res);
2177 38 : }
|
