TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * execParallel.c
4 : * Support routines for parallel execution.
5 : *
6 : * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
7 : * Portions Copyright (c) 1994, Regents of the University of California
8 : *
9 : * This file contains routines that are intended to support setting up,
10 : * using, and tearing down a ParallelContext from within the PostgreSQL
11 : * executor. The ParallelContext machinery will handle starting the
12 : * workers and ensuring that their state generally matches that of the
13 : * leader; see src/backend/access/transam/README.parallel for details.
14 : * However, we must save and restore relevant executor state, such as
15 : * any ParamListInfo associated with the query, buffer/WAL usage info, and
16 : * the actual plan to be passed down to the worker.
17 : *
18 : * IDENTIFICATION
19 : * src/backend/executor/execParallel.c
20 : *
21 : *-------------------------------------------------------------------------
22 : */
23 :
24 : #include "postgres.h"
25 :
26 : #include "executor/execParallel.h"
27 : #include "executor/executor.h"
28 : #include "executor/nodeAgg.h"
29 : #include "executor/nodeAppend.h"
30 : #include "executor/nodeBitmapHeapscan.h"
31 : #include "executor/nodeCustom.h"
32 : #include "executor/nodeForeignscan.h"
33 : #include "executor/nodeHash.h"
34 : #include "executor/nodeHashjoin.h"
35 : #include "executor/nodeIncrementalSort.h"
36 : #include "executor/nodeIndexonlyscan.h"
37 : #include "executor/nodeIndexscan.h"
38 : #include "executor/nodeMemoize.h"
39 : #include "executor/nodeSeqscan.h"
40 : #include "executor/nodeSort.h"
41 : #include "executor/nodeSubplan.h"
42 : #include "executor/tqueue.h"
43 : #include "jit/jit.h"
44 : #include "nodes/nodeFuncs.h"
45 : #include "pgstat.h"
46 : #include "storage/spin.h"
47 : #include "tcop/tcopprot.h"
48 : #include "utils/datum.h"
49 : #include "utils/dsa.h"
50 : #include "utils/lsyscache.h"
51 : #include "utils/memutils.h"
52 : #include "utils/snapmgr.h"
53 :
54 : /*
55 : * Magic numbers for parallel executor communication. We use constants
56 : * greater than any 32-bit integer here so that values < 2^32 can be used
57 : * by individual parallel nodes to store their own state.
58 : */
59 : #define PARALLEL_KEY_EXECUTOR_FIXED UINT64CONST(0xE000000000000001)
60 : #define PARALLEL_KEY_PLANNEDSTMT UINT64CONST(0xE000000000000002)
61 : #define PARALLEL_KEY_PARAMLISTINFO UINT64CONST(0xE000000000000003)
62 : #define PARALLEL_KEY_BUFFER_USAGE UINT64CONST(0xE000000000000004)
63 : #define PARALLEL_KEY_TUPLE_QUEUE UINT64CONST(0xE000000000000005)
64 : #define PARALLEL_KEY_INSTRUMENTATION UINT64CONST(0xE000000000000006)
65 : #define PARALLEL_KEY_DSA UINT64CONST(0xE000000000000007)
66 : #define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xE000000000000008)
67 : #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009)
68 : #define PARALLEL_KEY_WAL_USAGE UINT64CONST(0xE00000000000000A)
69 :
70 : #define PARALLEL_TUPLE_QUEUE_SIZE 65536
71 :
72 : /*
73 : * Fixed-size random stuff that we need to pass to parallel workers.
74 : */
75 : typedef struct FixedParallelExecutorState
76 : {
77 : int64 tuples_needed; /* tuple bound, see ExecSetTupleBound */
78 : dsa_pointer param_exec;
79 : int eflags;
80 : int jit_flags;
81 : } FixedParallelExecutorState;
82 :
83 : /*
84 : * DSM structure for accumulating per-PlanState instrumentation.
85 : *
86 : * instrument_options: Same meaning here as in instrument.c.
87 : *
88 : * instrument_offset: Offset, relative to the start of this structure,
89 : * of the first Instrumentation object. This will depend on the length of
90 : * the plan_node_id array.
91 : *
92 : * num_workers: Number of workers.
93 : *
94 : * num_plan_nodes: Number of plan nodes.
95 : *
96 : * plan_node_id: Array of plan nodes for which we are gathering instrumentation
97 : * from parallel workers. The length of this array is given by num_plan_nodes.
98 : */
99 : struct SharedExecutorInstrumentation
100 : {
101 : int instrument_options;
102 : int instrument_offset;
103 : int num_workers;
104 : int num_plan_nodes;
105 : int plan_node_id[FLEXIBLE_ARRAY_MEMBER];
106 : /* array of num_plan_nodes * num_workers Instrumentation objects follows */
107 : };
108 : #define GetInstrumentationArray(sei) \
109 : (AssertVariableIsOfTypeMacro(sei, SharedExecutorInstrumentation *), \
110 : (Instrumentation *) (((char *) sei) + sei->instrument_offset))
111 :
112 : /* Context object for ExecParallelEstimate. */
113 : typedef struct ExecParallelEstimateContext
114 : {
115 : ParallelContext *pcxt;
116 : int nnodes;
117 : } ExecParallelEstimateContext;
118 :
119 : /* Context object for ExecParallelInitializeDSM. */
120 : typedef struct ExecParallelInitializeDSMContext
121 : {
122 : ParallelContext *pcxt;
123 : SharedExecutorInstrumentation *instrumentation;
124 : int nnodes;
125 : } ExecParallelInitializeDSMContext;
126 :
127 : /* Helper functions that run in the parallel leader. */
128 : static char *ExecSerializePlan(Plan *plan, EState *estate);
129 : static bool ExecParallelEstimate(PlanState *planstate,
130 : ExecParallelEstimateContext *e);
131 : static bool ExecParallelInitializeDSM(PlanState *planstate,
132 : ExecParallelInitializeDSMContext *d);
133 : static shm_mq_handle **ExecParallelSetupTupleQueues(ParallelContext *pcxt,
134 : bool reinitialize);
135 : static bool ExecParallelReInitializeDSM(PlanState *planstate,
136 : ParallelContext *pcxt);
137 : static bool ExecParallelRetrieveInstrumentation(PlanState *planstate,
138 : SharedExecutorInstrumentation *instrumentation);
139 :
140 : /* Helper function that runs in the parallel worker. */
141 : static DestReceiver *ExecParallelGetReceiver(dsm_segment *seg, shm_toc *toc);
142 :
143 : /*
144 : * Create a serialized representation of the plan to be sent to each worker.
145 : */
146 : static char *
147 CBC 323 : ExecSerializePlan(Plan *plan, EState *estate)
148 : {
149 : PlannedStmt *pstmt;
150 : ListCell *lc;
151 :
152 : /* We can't scribble on the original plan, so make a copy. */
153 323 : plan = copyObject(plan);
154 :
155 : /*
156 : * The worker will start its own copy of the executor, and that copy will
157 : * insert a junk filter if the toplevel node has any resjunk entries. We
158 : * don't want that to happen, because while resjunk columns shouldn't be
159 : * sent back to the user, here the tuples are coming back to another
160 : * backend which may very well need them. So mutate the target list
161 : * accordingly. This is sort of a hack; there might be better ways to do
162 : * this...
163 : */
164 884 : foreach(lc, plan->targetlist)
165 : {
166 561 : TargetEntry *tle = lfirst_node(TargetEntry, lc);
167 :
168 561 : tle->resjunk = false;
169 : }
170 :
171 : /*
172 : * Create a dummy PlannedStmt. Most of the fields don't need to be valid
173 : * for our purposes, but the worker will need at least a minimal
174 : * PlannedStmt to start the executor.
175 : */
176 323 : pstmt = makeNode(PlannedStmt);
177 323 : pstmt->commandType = CMD_SELECT;
178 323 : pstmt->queryId = pgstat_get_my_query_id();
179 323 : pstmt->hasReturning = false;
180 323 : pstmt->hasModifyingCTE = false;
181 323 : pstmt->canSetTag = true;
182 323 : pstmt->transientPlan = false;
183 323 : pstmt->dependsOnRole = false;
184 323 : pstmt->parallelModeNeeded = false;
185 323 : pstmt->planTree = plan;
186 GNC 323 : pstmt->partPruneInfos = estate->es_part_prune_infos;
187 CBC 323 : pstmt->rtable = estate->es_range_table;
188 GNC 323 : pstmt->permInfos = estate->es_rteperminfos;
189 CBC 323 : pstmt->resultRelations = NIL;
190 323 : pstmt->appendRelations = NIL;
191 ECB :
192 : /*
193 : * Transfer only parallel-safe subplans, leaving a NULL "hole" in the list
194 : * for unsafe ones (so that the list indexes of the safe ones are
195 : * preserved). This positively ensures that the worker won't try to run,
196 : * or even do ExecInitNode on, an unsafe subplan. That's important to
197 : * protect, eg, non-parallel-aware FDWs from getting into trouble.
198 : */
199 GIC 323 : pstmt->subplans = NIL;
200 353 : foreach(lc, estate->es_plannedstmt->subplans)
201 ECB : {
202 CBC 30 : Plan *subplan = (Plan *) lfirst(lc);
203 :
204 30 : if (subplan && !subplan->parallel_safe)
205 GIC 27 : subplan = NULL;
206 CBC 30 : pstmt->subplans = lappend(pstmt->subplans, subplan);
207 ECB : }
208 :
209 GIC 323 : pstmt->rewindPlanIDs = NULL;
210 323 : pstmt->rowMarks = NIL;
211 CBC 323 : pstmt->relationOids = NIL;
212 323 : pstmt->invalItems = NIL; /* workers can't replan anyway... */
213 323 : pstmt->paramExecTypes = estate->es_plannedstmt->paramExecTypes;
214 323 : pstmt->utilityStmt = NULL;
215 323 : pstmt->stmt_location = -1;
216 323 : pstmt->stmt_len = -1;
217 ECB :
218 : /* Return serialized copy of our dummy PlannedStmt. */
219 GIC 323 : return nodeToString(pstmt);
220 : }
221 ECB :
222 : /*
223 : * Parallel-aware plan nodes (and occasionally others) may need some state
224 : * which is shared across all parallel workers. Before we size the DSM, give
225 : * them a chance to call shm_toc_estimate_chunk or shm_toc_estimate_keys on
226 : * &pcxt->estimator.
227 : *
228 : * While we're at it, count the number of PlanState nodes in the tree, so
229 : * we know how many Instrumentation structures we need.
230 : */
231 : static bool
232 GIC 1417 : ExecParallelEstimate(PlanState *planstate, ExecParallelEstimateContext *e)
233 : {
234 CBC 1417 : if (planstate == NULL)
235 UIC 0 : return false;
236 ECB :
237 EUB : /* Count this node. */
238 GIC 1417 : e->nnodes++;
239 :
240 CBC 1417 : switch (nodeTag(planstate))
241 : {
242 560 : case T_SeqScanState:
243 GIC 560 : if (planstate->plan->parallel_aware)
244 CBC 441 : ExecSeqScanEstimate((SeqScanState *) planstate,
245 ECB : e->pcxt);
246 CBC 560 : break;
247 GIC 144 : case T_IndexScanState:
248 CBC 144 : if (planstate->plan->parallel_aware)
249 6 : ExecIndexScanEstimate((IndexScanState *) planstate,
250 ECB : e->pcxt);
251 CBC 144 : break;
252 GIC 26 : case T_IndexOnlyScanState:
253 CBC 26 : if (planstate->plan->parallel_aware)
254 20 : ExecIndexOnlyScanEstimate((IndexOnlyScanState *) planstate,
255 ECB : e->pcxt);
256 CBC 26 : break;
257 UIC 0 : case T_ForeignScanState:
258 LBC 0 : if (planstate->plan->parallel_aware)
259 UBC 0 : ExecForeignScanEstimate((ForeignScanState *) planstate,
260 EUB : e->pcxt);
261 UBC 0 : break;
262 GIC 90 : case T_AppendState:
263 GBC 90 : if (planstate->plan->parallel_aware)
264 CBC 66 : ExecAppendEstimate((AppendState *) planstate,
265 ECB : e->pcxt);
266 CBC 90 : break;
267 UIC 0 : case T_CustomScanState:
268 LBC 0 : if (planstate->plan->parallel_aware)
269 UBC 0 : ExecCustomScanEstimate((CustomScanState *) planstate,
270 EUB : e->pcxt);
271 UBC 0 : break;
272 GIC 10 : case T_BitmapHeapScanState:
273 GBC 10 : if (planstate->plan->parallel_aware)
274 CBC 9 : ExecBitmapHeapEstimate((BitmapHeapScanState *) planstate,
275 ECB : e->pcxt);
276 CBC 10 : break;
277 GIC 93 : case T_HashJoinState:
278 CBC 93 : if (planstate->plan->parallel_aware)
279 57 : ExecHashJoinEstimate((HashJoinState *) planstate,
280 ECB : e->pcxt);
281 CBC 93 : break;
282 GIC 93 : case T_HashState:
283 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
284 CBC 93 : ExecHashEstimate((HashState *) planstate, e->pcxt);
285 GIC 93 : break;
286 CBC 70 : case T_SortState:
287 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
288 CBC 70 : ExecSortEstimate((SortState *) planstate, e->pcxt);
289 GIC 70 : break;
290 LBC 0 : case T_IncrementalSortState:
291 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
292 UBC 0 : ExecIncrementalSortEstimate((IncrementalSortState *) planstate, e->pcxt);
293 UIC 0 : break;
294 GBC 280 : case T_AggState:
295 EUB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
296 CBC 280 : ExecAggEstimate((AggState *) planstate, e->pcxt);
297 GIC 280 : break;
298 CBC 3 : case T_MemoizeState:
299 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
300 CBC 3 : ExecMemoizeEstimate((MemoizeState *) planstate, e->pcxt);
301 GIC 3 : break;
302 CBC 48 : default:
303 48 : break;
304 ECB : }
305 :
306 GIC 1417 : return planstate_tree_walker(planstate, ExecParallelEstimate, e);
307 : }
308 ECB :
309 : /*
310 : * Estimate the amount of space required to serialize the indicated parameters.
311 : */
312 : static Size
313 GIC 15 : EstimateParamExecSpace(EState *estate, Bitmapset *params)
314 : {
315 ECB : int paramid;
316 GIC 15 : Size sz = sizeof(int);
317 :
318 CBC 15 : paramid = -1;
319 GIC 33 : while ((paramid = bms_next_member(params, paramid)) >= 0)
320 ECB : {
321 : Oid typeOid;
322 : int16 typLen;
323 : bool typByVal;
324 : ParamExecData *prm;
325 :
326 GIC 18 : prm = &(estate->es_param_exec_vals[paramid]);
327 18 : typeOid = list_nth_oid(estate->es_plannedstmt->paramExecTypes,
328 ECB : paramid);
329 :
330 GIC 18 : sz = add_size(sz, sizeof(int)); /* space for paramid */
331 :
332 ECB : /* space for datum/isnull */
333 GIC 18 : if (OidIsValid(typeOid))
334 18 : get_typlenbyval(typeOid, &typLen, &typByVal);
335 ECB : else
336 : {
337 : /* If no type OID, assume by-value, like copyParamList does. */
338 UIC 0 : typLen = sizeof(Datum);
339 0 : typByVal = true;
340 EUB : }
341 GBC 18 : sz = add_size(sz,
342 GIC 18 : datumEstimateSpace(prm->value, prm->isnull,
343 ECB : typByVal, typLen));
344 : }
345 GIC 15 : return sz;
346 : }
347 ECB :
348 : /*
349 : * Serialize specified PARAM_EXEC parameters.
350 : *
351 : * We write the number of parameters first, as a 4-byte integer, and then
352 : * write details for each parameter in turn. The details for each parameter
353 : * consist of a 4-byte paramid (location of param in execution time internal
354 : * parameter array) and then the datum as serialized by datumSerialize().
355 : */
356 : static dsa_pointer
357 GIC 15 : SerializeParamExecParams(EState *estate, Bitmapset *params, dsa_area *area)
358 : {
359 ECB : Size size;
360 : int nparams;
361 : int paramid;
362 : ParamExecData *prm;
363 : dsa_pointer handle;
364 : char *start_address;
365 :
366 : /* Allocate enough space for the current parameter values. */
367 GIC 15 : size = EstimateParamExecSpace(estate, params);
368 15 : handle = dsa_allocate(area, size);
369 CBC 15 : start_address = dsa_get_address(area, handle);
370 ECB :
371 : /* First write the number of parameters as a 4-byte integer. */
372 GIC 15 : nparams = bms_num_members(params);
373 15 : memcpy(start_address, &nparams, sizeof(int));
374 CBC 15 : start_address += sizeof(int);
375 ECB :
376 : /* Write details for each parameter in turn. */
377 GIC 15 : paramid = -1;
378 33 : while ((paramid = bms_next_member(params, paramid)) >= 0)
379 ECB : {
380 : Oid typeOid;
381 : int16 typLen;
382 : bool typByVal;
383 :
384 GIC 18 : prm = &(estate->es_param_exec_vals[paramid]);
385 18 : typeOid = list_nth_oid(estate->es_plannedstmt->paramExecTypes,
386 ECB : paramid);
387 :
388 : /* Write paramid. */
389 GIC 18 : memcpy(start_address, ¶mid, sizeof(int));
390 18 : start_address += sizeof(int);
391 ECB :
392 : /* Write datum/isnull */
393 GIC 18 : if (OidIsValid(typeOid))
394 18 : get_typlenbyval(typeOid, &typLen, &typByVal);
395 ECB : else
396 : {
397 : /* If no type OID, assume by-value, like copyParamList does. */
398 UIC 0 : typLen = sizeof(Datum);
399 0 : typByVal = true;
400 EUB : }
401 GBC 18 : datumSerialize(prm->value, prm->isnull, typByVal, typLen,
402 : &start_address);
403 ECB : }
404 :
405 GIC 15 : return handle;
406 : }
407 ECB :
408 : /*
409 : * Restore specified PARAM_EXEC parameters.
410 : */
411 : static void
412 GIC 35 : RestoreParamExecParams(char *start_address, EState *estate)
413 : {
414 ECB : int nparams;
415 : int i;
416 : int paramid;
417 :
418 GIC 35 : memcpy(&nparams, start_address, sizeof(int));
419 35 : start_address += sizeof(int);
420 ECB :
421 CBC 75 : for (i = 0; i < nparams; i++)
422 : {
423 ECB : ParamExecData *prm;
424 :
425 : /* Read paramid */
426 GIC 40 : memcpy(¶mid, start_address, sizeof(int));
427 40 : start_address += sizeof(int);
428 CBC 40 : prm = &(estate->es_param_exec_vals[paramid]);
429 ECB :
430 : /* Read datum/isnull. */
431 GIC 40 : prm->value = datumRestore(&start_address, &prm->isnull);
432 40 : prm->execPlan = NULL;
433 ECB : }
434 CBC 35 : }
435 :
436 ECB : /*
437 : * Initialize the dynamic shared memory segment that will be used to control
438 : * parallel execution.
439 : */
440 : static bool
441 GIC 1417 : ExecParallelInitializeDSM(PlanState *planstate,
442 : ExecParallelInitializeDSMContext *d)
443 ECB : {
444 GIC 1417 : if (planstate == NULL)
445 UIC 0 : return false;
446 ECB :
447 EUB : /* If instrumentation is enabled, initialize slot for this node. */
448 GIC 1417 : if (d->instrumentation != NULL)
449 507 : d->instrumentation->plan_node_id[d->nnodes] =
450 CBC 507 : planstate->plan->plan_node_id;
451 ECB :
452 : /* Count this node. */
453 GIC 1417 : d->nnodes++;
454 :
455 ECB : /*
456 : * Call initializers for DSM-using plan nodes.
457 : *
458 : * Most plan nodes won't do anything here, but plan nodes that allocated
459 : * DSM may need to initialize shared state in the DSM before parallel
460 : * workers are launched. They can allocate the space they previously
461 : * estimated using shm_toc_allocate, and add the keys they previously
462 : * estimated using shm_toc_insert, in each case targeting pcxt->toc.
463 : */
464 GIC 1417 : switch (nodeTag(planstate))
465 : {
466 CBC 560 : case T_SeqScanState:
467 GIC 560 : if (planstate->plan->parallel_aware)
468 CBC 441 : ExecSeqScanInitializeDSM((SeqScanState *) planstate,
469 ECB : d->pcxt);
470 CBC 560 : break;
471 GIC 144 : case T_IndexScanState:
472 CBC 144 : if (planstate->plan->parallel_aware)
473 6 : ExecIndexScanInitializeDSM((IndexScanState *) planstate,
474 ECB : d->pcxt);
475 CBC 144 : break;
476 GIC 26 : case T_IndexOnlyScanState:
477 CBC 26 : if (planstate->plan->parallel_aware)
478 20 : ExecIndexOnlyScanInitializeDSM((IndexOnlyScanState *) planstate,
479 ECB : d->pcxt);
480 CBC 26 : break;
481 UIC 0 : case T_ForeignScanState:
482 LBC 0 : if (planstate->plan->parallel_aware)
483 UBC 0 : ExecForeignScanInitializeDSM((ForeignScanState *) planstate,
484 EUB : d->pcxt);
485 UBC 0 : break;
486 GIC 90 : case T_AppendState:
487 GBC 90 : if (planstate->plan->parallel_aware)
488 CBC 66 : ExecAppendInitializeDSM((AppendState *) planstate,
489 ECB : d->pcxt);
490 CBC 90 : break;
491 UIC 0 : case T_CustomScanState:
492 LBC 0 : if (planstate->plan->parallel_aware)
493 UBC 0 : ExecCustomScanInitializeDSM((CustomScanState *) planstate,
494 EUB : d->pcxt);
495 UBC 0 : break;
496 GIC 10 : case T_BitmapHeapScanState:
497 GBC 10 : if (planstate->plan->parallel_aware)
498 CBC 9 : ExecBitmapHeapInitializeDSM((BitmapHeapScanState *) planstate,
499 ECB : d->pcxt);
500 CBC 10 : break;
501 GIC 93 : case T_HashJoinState:
502 CBC 93 : if (planstate->plan->parallel_aware)
503 57 : ExecHashJoinInitializeDSM((HashJoinState *) planstate,
504 ECB : d->pcxt);
505 CBC 93 : break;
506 GIC 93 : case T_HashState:
507 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
508 CBC 93 : ExecHashInitializeDSM((HashState *) planstate, d->pcxt);
509 GIC 93 : break;
510 CBC 70 : case T_SortState:
511 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
512 CBC 70 : ExecSortInitializeDSM((SortState *) planstate, d->pcxt);
513 GIC 70 : break;
514 LBC 0 : case T_IncrementalSortState:
515 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
516 UBC 0 : ExecIncrementalSortInitializeDSM((IncrementalSortState *) planstate, d->pcxt);
517 UIC 0 : break;
518 GBC 280 : case T_AggState:
519 EUB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
520 CBC 280 : ExecAggInitializeDSM((AggState *) planstate, d->pcxt);
521 GIC 280 : break;
522 CBC 3 : case T_MemoizeState:
523 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
524 CBC 3 : ExecMemoizeInitializeDSM((MemoizeState *) planstate, d->pcxt);
525 GIC 3 : break;
526 CBC 48 : default:
527 48 : break;
528 ECB : }
529 :
530 GIC 1417 : return planstate_tree_walker(planstate, ExecParallelInitializeDSM, d);
531 : }
532 ECB :
533 : /*
534 : * It sets up the response queues for backend workers to return tuples
535 : * to the main backend and start the workers.
536 : */
537 : static shm_mq_handle **
538 GIC 452 : ExecParallelSetupTupleQueues(ParallelContext *pcxt, bool reinitialize)
539 : {
540 ECB : shm_mq_handle **responseq;
541 : char *tqueuespace;
542 : int i;
543 :
544 : /* Skip this if no workers. */
545 GIC 452 : if (pcxt->nworkers == 0)
546 UIC 0 : return NULL;
547 ECB :
548 EUB : /* Allocate memory for shared memory queue handles. */
549 : responseq = (shm_mq_handle **)
550 GIC 452 : palloc(pcxt->nworkers * sizeof(shm_mq_handle *));
551 :
552 ECB : /*
553 : * If not reinitializing, allocate space from the DSM for the queues;
554 : * otherwise, find the already allocated space.
555 : */
556 GIC 452 : if (!reinitialize)
557 : tqueuespace =
558 CBC 323 : shm_toc_allocate(pcxt->toc,
559 : mul_size(PARALLEL_TUPLE_QUEUE_SIZE,
560 323 : pcxt->nworkers));
561 : else
562 129 : tqueuespace = shm_toc_lookup(pcxt->toc, PARALLEL_KEY_TUPLE_QUEUE, false);
563 :
564 ECB : /* Create the queues, and become the receiver for each. */
565 GIC 1700 : for (i = 0; i < pcxt->nworkers; ++i)
566 : {
567 ECB : shm_mq *mq;
568 :
569 GIC 1248 : mq = shm_mq_create(tqueuespace +
570 1248 : ((Size) i) * PARALLEL_TUPLE_QUEUE_SIZE,
571 ECB : (Size) PARALLEL_TUPLE_QUEUE_SIZE);
572 :
573 GIC 1248 : shm_mq_set_receiver(mq, MyProc);
574 1248 : responseq[i] = shm_mq_attach(mq, pcxt->seg, NULL);
575 ECB : }
576 :
577 : /* Add array of queues to shm_toc, so others can find it. */
578 GIC 452 : if (!reinitialize)
579 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_TUPLE_QUEUE, tqueuespace);
580 ECB :
581 : /* Return array of handles. */
582 GIC 452 : return responseq;
583 : }
584 ECB :
585 : /*
586 : * Sets up the required infrastructure for backend workers to perform
587 : * execution and return results to the main backend.
588 : */
589 : ParallelExecutorInfo *
590 GIC 323 : ExecInitParallelPlan(PlanState *planstate, EState *estate,
591 : Bitmapset *sendParams, int nworkers,
592 ECB : int64 tuples_needed)
593 : {
594 : ParallelExecutorInfo *pei;
595 : ParallelContext *pcxt;
596 : ExecParallelEstimateContext e;
597 : ExecParallelInitializeDSMContext d;
598 : FixedParallelExecutorState *fpes;
599 : char *pstmt_data;
600 : char *pstmt_space;
601 : char *paramlistinfo_space;
602 : BufferUsage *bufusage_space;
603 : WalUsage *walusage_space;
604 GIC 323 : SharedExecutorInstrumentation *instrumentation = NULL;
605 323 : SharedJitInstrumentation *jit_instrumentation = NULL;
606 ECB : int pstmt_len;
607 : int paramlistinfo_len;
608 GIC 323 : int instrumentation_len = 0;
609 323 : int jit_instrumentation_len = 0;
610 CBC 323 : int instrument_offset = 0;
611 323 : Size dsa_minsize = dsa_minimum_size();
612 ECB : char *query_string;
613 : int query_len;
614 :
615 : /*
616 : * Force any initplan outputs that we're going to pass to workers to be
617 : * evaluated, if they weren't already.
618 : *
619 : * For simplicity, we use the EState's per-output-tuple ExprContext here.
620 : * That risks intra-query memory leakage, since we might pass through here
621 : * many times before that ExprContext gets reset; but ExecSetParamPlan
622 : * doesn't normally leak any memory in the context (see its comments), so
623 : * it doesn't seem worth complicating this function's API to pass it a
624 : * shorter-lived ExprContext. This might need to change someday.
625 : */
626 GIC 323 : ExecSetParamPlanMulti(sendParams, GetPerTupleExprContext(estate));
627 :
628 ECB : /* Allocate object for return value. */
629 GIC 323 : pei = palloc0(sizeof(ParallelExecutorInfo));
630 323 : pei->finished = false;
631 CBC 323 : pei->planstate = planstate;
632 ECB :
633 : /* Fix up and serialize plan to be sent to workers. */
634 GIC 323 : pstmt_data = ExecSerializePlan(planstate->plan, estate);
635 :
636 ECB : /* Create a parallel context. */
637 GIC 323 : pcxt = CreateParallelContext("postgres", "ParallelQueryMain", nworkers);
638 323 : pei->pcxt = pcxt;
639 ECB :
640 : /*
641 : * Before telling the parallel context to create a dynamic shared memory
642 : * segment, we need to figure out how big it should be. Estimate space
643 : * for the various things we need to store.
644 : */
645 :
646 : /* Estimate space for fixed-size state. */
647 GIC 323 : shm_toc_estimate_chunk(&pcxt->estimator,
648 : sizeof(FixedParallelExecutorState));
649 CBC 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
650 :
651 ECB : /* Estimate space for query text. */
652 GIC 323 : query_len = strlen(estate->es_sourceText);
653 323 : shm_toc_estimate_chunk(&pcxt->estimator, query_len + 1);
654 CBC 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
655 ECB :
656 : /* Estimate space for serialized PlannedStmt. */
657 GIC 323 : pstmt_len = strlen(pstmt_data) + 1;
658 323 : shm_toc_estimate_chunk(&pcxt->estimator, pstmt_len);
659 CBC 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
660 ECB :
661 : /* Estimate space for serialized ParamListInfo. */
662 GIC 323 : paramlistinfo_len = EstimateParamListSpace(estate->es_param_list_info);
663 323 : shm_toc_estimate_chunk(&pcxt->estimator, paramlistinfo_len);
664 CBC 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
665 ECB :
666 : /*
667 : * Estimate space for BufferUsage.
668 : *
669 : * If EXPLAIN is not in use and there are no extensions loaded that care,
670 : * we could skip this. But we have no way of knowing whether anyone's
671 : * looking at pgBufferUsage, so do it unconditionally.
672 : */
673 GIC 323 : shm_toc_estimate_chunk(&pcxt->estimator,
674 : mul_size(sizeof(BufferUsage), pcxt->nworkers));
675 CBC 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
676 :
677 ECB : /*
678 : * Same thing for WalUsage.
679 : */
680 GIC 323 : shm_toc_estimate_chunk(&pcxt->estimator,
681 : mul_size(sizeof(WalUsage), pcxt->nworkers));
682 CBC 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
683 :
684 ECB : /* Estimate space for tuple queues. */
685 GIC 323 : shm_toc_estimate_chunk(&pcxt->estimator,
686 : mul_size(PARALLEL_TUPLE_QUEUE_SIZE, pcxt->nworkers));
687 CBC 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
688 :
689 ECB : /*
690 : * Give parallel-aware nodes a chance to add to the estimates, and get a
691 : * count of how many PlanState nodes there are.
692 : */
693 GIC 323 : e.pcxt = pcxt;
694 323 : e.nnodes = 0;
695 CBC 323 : ExecParallelEstimate(planstate, &e);
696 ECB :
697 : /* Estimate space for instrumentation, if required. */
698 GIC 323 : if (estate->es_instrument)
699 : {
700 CBC 93 : instrumentation_len =
701 : offsetof(SharedExecutorInstrumentation, plan_node_id) +
702 93 : sizeof(int) * e.nnodes;
703 GIC 93 : instrumentation_len = MAXALIGN(instrumentation_len);
704 CBC 93 : instrument_offset = instrumentation_len;
705 93 : instrumentation_len +=
706 93 : mul_size(sizeof(Instrumentation),
707 93 : mul_size(e.nnodes, nworkers));
708 93 : shm_toc_estimate_chunk(&pcxt->estimator, instrumentation_len);
709 93 : shm_toc_estimate_keys(&pcxt->estimator, 1);
710 ECB :
711 : /* Estimate space for JIT instrumentation, if required. */
712 GIC 93 : if (estate->es_jit_flags != PGJIT_NONE)
713 : {
714 CBC 12 : jit_instrumentation_len =
715 GIC 12 : offsetof(SharedJitInstrumentation, jit_instr) +
716 ECB : sizeof(JitInstrumentation) * nworkers;
717 CBC 12 : shm_toc_estimate_chunk(&pcxt->estimator, jit_instrumentation_len);
718 GIC 12 : shm_toc_estimate_keys(&pcxt->estimator, 1);
719 ECB : }
720 : }
721 :
722 : /* Estimate space for DSA area. */
723 GIC 323 : shm_toc_estimate_chunk(&pcxt->estimator, dsa_minsize);
724 323 : shm_toc_estimate_keys(&pcxt->estimator, 1);
725 ECB :
726 : /* Everyone's had a chance to ask for space, so now create the DSM. */
727 GIC 323 : InitializeParallelDSM(pcxt);
728 :
729 ECB : /*
730 : * OK, now we have a dynamic shared memory segment, and it should be big
731 : * enough to store all of the data we estimated we would want to put into
732 : * it, plus whatever general stuff (not specifically executor-related) the
733 : * ParallelContext itself needs to store there. None of the space we
734 : * asked for has been allocated or initialized yet, though, so do that.
735 : */
736 :
737 : /* Store fixed-size state. */
738 GIC 323 : fpes = shm_toc_allocate(pcxt->toc, sizeof(FixedParallelExecutorState));
739 323 : fpes->tuples_needed = tuples_needed;
740 CBC 323 : fpes->param_exec = InvalidDsaPointer;
741 323 : fpes->eflags = estate->es_top_eflags;
742 323 : fpes->jit_flags = estate->es_jit_flags;
743 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, fpes);
744 ECB :
745 : /* Store query string */
746 GIC 323 : query_string = shm_toc_allocate(pcxt->toc, query_len + 1);
747 323 : memcpy(query_string, estate->es_sourceText, query_len + 1);
748 CBC 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_QUERY_TEXT, query_string);
749 ECB :
750 : /* Store serialized PlannedStmt. */
751 GIC 323 : pstmt_space = shm_toc_allocate(pcxt->toc, pstmt_len);
752 323 : memcpy(pstmt_space, pstmt_data, pstmt_len);
753 CBC 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_PLANNEDSTMT, pstmt_space);
754 ECB :
755 : /* Store serialized ParamListInfo. */
756 GIC 323 : paramlistinfo_space = shm_toc_allocate(pcxt->toc, paramlistinfo_len);
757 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_PARAMLISTINFO, paramlistinfo_space);
758 CBC 323 : SerializeParamList(estate->es_param_list_info, ¶mlistinfo_space);
759 ECB :
760 : /* Allocate space for each worker's BufferUsage; no need to initialize. */
761 GIC 323 : bufusage_space = shm_toc_allocate(pcxt->toc,
762 323 : mul_size(sizeof(BufferUsage), pcxt->nworkers));
763 CBC 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_BUFFER_USAGE, bufusage_space);
764 323 : pei->buffer_usage = bufusage_space;
765 ECB :
766 : /* Same for WalUsage. */
767 GIC 323 : walusage_space = shm_toc_allocate(pcxt->toc,
768 323 : mul_size(sizeof(WalUsage), pcxt->nworkers));
769 CBC 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_WAL_USAGE, walusage_space);
770 323 : pei->wal_usage = walusage_space;
771 ECB :
772 : /* Set up the tuple queues that the workers will write into. */
773 GIC 323 : pei->tqueue = ExecParallelSetupTupleQueues(pcxt, false);
774 :
775 ECB : /* We don't need the TupleQueueReaders yet, though. */
776 GIC 323 : pei->reader = NULL;
777 :
778 ECB : /*
779 : * If instrumentation options were supplied, allocate space for the data.
780 : * It only gets partially initialized here; the rest happens during
781 : * ExecParallelInitializeDSM.
782 : */
783 GIC 323 : if (estate->es_instrument)
784 : {
785 ECB : Instrumentation *instrument;
786 : int i;
787 :
788 GIC 93 : instrumentation = shm_toc_allocate(pcxt->toc, instrumentation_len);
789 93 : instrumentation->instrument_options = estate->es_instrument;
790 CBC 93 : instrumentation->instrument_offset = instrument_offset;
791 93 : instrumentation->num_workers = nworkers;
792 93 : instrumentation->num_plan_nodes = e.nnodes;
793 93 : instrument = GetInstrumentationArray(instrumentation);
794 918 : for (i = 0; i < nworkers * e.nnodes; ++i)
795 825 : InstrInit(&instrument[i], estate->es_instrument);
796 93 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_INSTRUMENTATION,
797 ECB : instrumentation);
798 CBC 93 : pei->instrumentation = instrumentation;
799 :
800 93 : if (estate->es_jit_flags != PGJIT_NONE)
801 : {
802 12 : jit_instrumentation = shm_toc_allocate(pcxt->toc,
803 : jit_instrumentation_len);
804 12 : jit_instrumentation->num_workers = nworkers;
805 GIC 12 : memset(jit_instrumentation->jit_instr, 0,
806 ECB : sizeof(JitInstrumentation) * nworkers);
807 CBC 12 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_JIT_INSTRUMENTATION,
808 : jit_instrumentation);
809 12 : pei->jit_instrumentation = jit_instrumentation;
810 : }
811 ECB : }
812 :
813 : /*
814 : * Create a DSA area that can be used by the leader and all workers.
815 : * (However, if we failed to create a DSM and are using private memory
816 : * instead, then skip this.)
817 : */
818 GIC 323 : if (pcxt->seg != NULL)
819 : {
820 ECB : char *area_space;
821 :
822 GIC 323 : area_space = shm_toc_allocate(pcxt->toc, dsa_minsize);
823 323 : shm_toc_insert(pcxt->toc, PARALLEL_KEY_DSA, area_space);
824 CBC 323 : pei->area = dsa_create_in_place(area_space, dsa_minsize,
825 ECB : LWTRANCHE_PARALLEL_QUERY_DSA,
826 : pcxt->seg);
827 :
828 : /*
829 : * Serialize parameters, if any, using DSA storage. We don't dare use
830 : * the main parallel query DSM for this because we might relaunch
831 : * workers after the values have changed (and thus the amount of
832 : * storage required has changed).
833 : */
834 GIC 323 : if (!bms_is_empty(sendParams))
835 : {
836 CBC 15 : pei->param_exec = SerializeParamExecParams(estate, sendParams,
837 : pei->area);
838 15 : fpes->param_exec = pei->param_exec;
839 : }
840 ECB : }
841 :
842 : /*
843 : * Give parallel-aware nodes a chance to initialize their shared data.
844 : * This also initializes the elements of instrumentation->ps_instrument,
845 : * if it exists.
846 : */
847 GIC 323 : d.pcxt = pcxt;
848 323 : d.instrumentation = instrumentation;
849 CBC 323 : d.nnodes = 0;
850 ECB :
851 : /* Install our DSA area while initializing the plan. */
852 GIC 323 : estate->es_query_dsa = pei->area;
853 323 : ExecParallelInitializeDSM(planstate, &d);
854 CBC 323 : estate->es_query_dsa = NULL;
855 ECB :
856 : /*
857 : * Make sure that the world hasn't shifted under our feet. This could
858 : * probably just be an Assert(), but let's be conservative for now.
859 : */
860 GIC 323 : if (e.nnodes != d.nnodes)
861 UIC 0 : elog(ERROR, "inconsistent count of PlanState nodes");
862 ECB :
863 EUB : /* OK, we're ready to rock and roll. */
864 GIC 323 : return pei;
865 : }
866 ECB :
867 : /*
868 : * Set up tuple queue readers to read the results of a parallel subplan.
869 : *
870 : * This is separate from ExecInitParallelPlan() because we can launch the
871 : * worker processes and let them start doing something before we do this.
872 : */
873 : void
874 GIC 443 : ExecParallelCreateReaders(ParallelExecutorInfo *pei)
875 : {
876 CBC 443 : int nworkers = pei->pcxt->nworkers_launched;
877 : int i;
878 ECB :
879 GIC 443 : Assert(pei->reader == NULL);
880 :
881 CBC 443 : if (nworkers > 0)
882 : {
883 443 : pei->reader = (TupleQueueReader **)
884 GIC 443 : palloc(nworkers * sizeof(TupleQueueReader *));
885 ECB :
886 CBC 1653 : for (i = 0; i < nworkers; i++)
887 : {
888 1210 : shm_mq_set_handle(pei->tqueue[i],
889 GIC 1210 : pei->pcxt->worker[i].bgwhandle);
890 CBC 1210 : pei->reader[i] = CreateTupleQueueReader(pei->tqueue[i]);
891 ECB : }
892 : }
893 GIC 443 : }
894 :
895 ECB : /*
896 : * Re-initialize the parallel executor shared memory state before launching
897 : * a fresh batch of workers.
898 : */
899 : void
900 GIC 129 : ExecParallelReinitialize(PlanState *planstate,
901 : ParallelExecutorInfo *pei,
902 ECB : Bitmapset *sendParams)
903 : {
904 GIC 129 : EState *estate = planstate->state;
905 : FixedParallelExecutorState *fpes;
906 ECB :
907 : /* Old workers must already be shut down */
908 GIC 129 : Assert(pei->finished);
909 :
910 ECB : /*
911 : * Force any initplan outputs that we're going to pass to workers to be
912 : * evaluated, if they weren't already (see comments in
913 : * ExecInitParallelPlan).
914 : */
915 GIC 129 : ExecSetParamPlanMulti(sendParams, GetPerTupleExprContext(estate));
916 :
917 CBC 129 : ReinitializeParallelDSM(pei->pcxt);
918 GIC 129 : pei->tqueue = ExecParallelSetupTupleQueues(pei->pcxt, true);
919 CBC 129 : pei->reader = NULL;
920 129 : pei->finished = false;
921 ECB :
922 CBC 129 : fpes = shm_toc_lookup(pei->pcxt->toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
923 :
924 ECB : /* Free any serialized parameters from the last round. */
925 GIC 129 : if (DsaPointerIsValid(fpes->param_exec))
926 : {
927 LBC 0 : dsa_free(pei->area, fpes->param_exec);
928 UIC 0 : fpes->param_exec = InvalidDsaPointer;
929 EUB : }
930 :
931 : /* Serialize current parameter values if required. */
932 GIC 129 : if (!bms_is_empty(sendParams))
933 : {
934 LBC 0 : pei->param_exec = SerializeParamExecParams(estate, sendParams,
935 : pei->area);
936 UBC 0 : fpes->param_exec = pei->param_exec;
937 : }
938 EUB :
939 : /* Traverse plan tree and let each child node reset associated state. */
940 GIC 129 : estate->es_query_dsa = pei->area;
941 129 : ExecParallelReInitializeDSM(planstate, pei->pcxt);
942 CBC 129 : estate->es_query_dsa = NULL;
943 129 : }
944 ECB :
945 : /*
946 : * Traverse plan tree to reinitialize per-node dynamic shared memory state
947 : */
948 : static bool
949 GIC 333 : ExecParallelReInitializeDSM(PlanState *planstate,
950 : ParallelContext *pcxt)
951 ECB : {
952 GIC 333 : if (planstate == NULL)
953 UIC 0 : return false;
954 ECB :
955 EUB : /*
956 : * Call reinitializers for DSM-using plan nodes.
957 : */
958 GIC 333 : switch (nodeTag(planstate))
959 : {
960 CBC 138 : case T_SeqScanState:
961 GIC 138 : if (planstate->plan->parallel_aware)
962 CBC 114 : ExecSeqScanReInitializeDSM((SeqScanState *) planstate,
963 ECB : pcxt);
964 CBC 138 : break;
965 GIC 6 : case T_IndexScanState:
966 CBC 6 : if (planstate->plan->parallel_aware)
967 6 : ExecIndexScanReInitializeDSM((IndexScanState *) planstate,
968 ECB : pcxt);
969 CBC 6 : break;
970 GIC 6 : case T_IndexOnlyScanState:
971 CBC 6 : if (planstate->plan->parallel_aware)
972 6 : ExecIndexOnlyScanReInitializeDSM((IndexOnlyScanState *) planstate,
973 ECB : pcxt);
974 CBC 6 : break;
975 UIC 0 : case T_ForeignScanState:
976 LBC 0 : if (planstate->plan->parallel_aware)
977 UBC 0 : ExecForeignScanReInitializeDSM((ForeignScanState *) planstate,
978 EUB : pcxt);
979 UBC 0 : break;
980 UIC 0 : case T_AppendState:
981 UBC 0 : if (planstate->plan->parallel_aware)
982 0 : ExecAppendReInitializeDSM((AppendState *) planstate, pcxt);
983 0 : break;
984 0 : case T_CustomScanState:
985 0 : if (planstate->plan->parallel_aware)
986 0 : ExecCustomScanReInitializeDSM((CustomScanState *) planstate,
987 EUB : pcxt);
988 UBC 0 : break;
989 GIC 27 : case T_BitmapHeapScanState:
990 GBC 27 : if (planstate->plan->parallel_aware)
991 CBC 27 : ExecBitmapHeapReInitializeDSM((BitmapHeapScanState *) planstate,
992 ECB : pcxt);
993 CBC 27 : break;
994 GIC 48 : case T_HashJoinState:
995 CBC 48 : if (planstate->plan->parallel_aware)
996 24 : ExecHashJoinReInitializeDSM((HashJoinState *) planstate,
997 ECB : pcxt);
998 CBC 48 : break;
999 GIC 63 : case T_HashState:
1000 ECB : case T_SortState:
1001 : case T_IncrementalSortState:
1002 : case T_MemoizeState:
1003 : /* these nodes have DSM state, but no reinitialization is required */
1004 GIC 63 : break;
1005 :
1006 CBC 45 : default:
1007 GIC 45 : break;
1008 ECB : }
1009 :
1010 GIC 333 : return planstate_tree_walker(planstate, ExecParallelReInitializeDSM, pcxt);
1011 : }
1012 ECB :
1013 : /*
1014 : * Copy instrumentation information about this node and its descendants from
1015 : * dynamic shared memory.
1016 : */
1017 : static bool
1018 GIC 507 : ExecParallelRetrieveInstrumentation(PlanState *planstate,
1019 : SharedExecutorInstrumentation *instrumentation)
1020 ECB : {
1021 : Instrumentation *instrument;
1022 : int i;
1023 : int n;
1024 : int ibytes;
1025 GIC 507 : int plan_node_id = planstate->plan->plan_node_id;
1026 : MemoryContext oldcontext;
1027 ECB :
1028 : /* Find the instrumentation for this node. */
1029 GIC 2235 : for (i = 0; i < instrumentation->num_plan_nodes; ++i)
1030 2235 : if (instrumentation->plan_node_id[i] == plan_node_id)
1031 CBC 507 : break;
1032 507 : if (i >= instrumentation->num_plan_nodes)
1033 LBC 0 : elog(ERROR, "plan node %d not found", plan_node_id);
1034 ECB :
1035 EUB : /* Accumulate the statistics from all workers. */
1036 GIC 507 : instrument = GetInstrumentationArray(instrumentation);
1037 507 : instrument += i * instrumentation->num_workers;
1038 CBC 1332 : for (n = 0; n < instrumentation->num_workers; ++n)
1039 825 : InstrAggNode(planstate->instrument, &instrument[n]);
1040 ECB :
1041 : /*
1042 : * Also store the per-worker detail.
1043 : *
1044 : * Worker instrumentation should be allocated in the same context as the
1045 : * regular instrumentation information, which is the per-query context.
1046 : * Switch into per-query memory context.
1047 : */
1048 GIC 507 : oldcontext = MemoryContextSwitchTo(planstate->state->es_query_cxt);
1049 507 : ibytes = mul_size(instrumentation->num_workers, sizeof(Instrumentation));
1050 CBC 507 : planstate->worker_instrument =
1051 507 : palloc(ibytes + offsetof(WorkerInstrumentation, instrument));
1052 507 : MemoryContextSwitchTo(oldcontext);
1053 ECB :
1054 CBC 507 : planstate->worker_instrument->num_workers = instrumentation->num_workers;
1055 GIC 507 : memcpy(&planstate->worker_instrument->instrument, instrument, ibytes);
1056 ECB :
1057 : /* Perform any node-type-specific work that needs to be done. */
1058 GIC 507 : switch (nodeTag(planstate))
1059 : {
1060 CBC 6 : case T_SortState:
1061 GIC 6 : ExecSortRetrieveInstrumentation((SortState *) planstate);
1062 CBC 6 : break;
1063 LBC 0 : case T_IncrementalSortState:
1064 0 : ExecIncrementalSortRetrieveInstrumentation((IncrementalSortState *) planstate);
1065 UBC 0 : break;
1066 GBC 42 : case T_HashState:
1067 42 : ExecHashRetrieveInstrumentation((HashState *) planstate);
1068 CBC 42 : break;
1069 54 : case T_AggState:
1070 54 : ExecAggRetrieveInstrumentation((AggState *) planstate);
1071 54 : break;
1072 LBC 0 : case T_MemoizeState:
1073 0 : ExecMemoizeRetrieveInstrumentation((MemoizeState *) planstate);
1074 UBC 0 : break;
1075 GBC 405 : default:
1076 405 : break;
1077 ECB : }
1078 :
1079 GIC 507 : return planstate_tree_walker(planstate, ExecParallelRetrieveInstrumentation,
1080 : instrumentation);
1081 ECB : }
1082 :
1083 : /*
1084 : * Add up the workers' JIT instrumentation from dynamic shared memory.
1085 : */
1086 : static void
1087 GIC 12 : ExecParallelRetrieveJitInstrumentation(PlanState *planstate,
1088 : SharedJitInstrumentation *shared_jit)
1089 ECB : {
1090 : JitInstrumentation *combined;
1091 : int ibytes;
1092 :
1093 : int n;
1094 :
1095 : /*
1096 : * Accumulate worker JIT instrumentation into the combined JIT
1097 : * instrumentation, allocating it if required.
1098 : */
1099 GIC 12 : if (!planstate->state->es_jit_worker_instr)
1100 12 : planstate->state->es_jit_worker_instr =
1101 CBC 12 : MemoryContextAllocZero(planstate->state->es_query_cxt, sizeof(JitInstrumentation));
1102 12 : combined = planstate->state->es_jit_worker_instr;
1103 ECB :
1104 : /* Accumulate all the workers' instrumentations. */
1105 GIC 36 : for (n = 0; n < shared_jit->num_workers; ++n)
1106 24 : InstrJitAgg(combined, &shared_jit->jit_instr[n]);
1107 ECB :
1108 : /*
1109 : * Store the per-worker detail.
1110 : *
1111 : * Similar to ExecParallelRetrieveInstrumentation(), allocate the
1112 : * instrumentation in per-query context.
1113 : */
1114 GIC 12 : ibytes = offsetof(SharedJitInstrumentation, jit_instr)
1115 12 : + mul_size(shared_jit->num_workers, sizeof(JitInstrumentation));
1116 CBC 12 : planstate->worker_jit_instrument =
1117 12 : MemoryContextAlloc(planstate->state->es_query_cxt, ibytes);
1118 ECB :
1119 CBC 12 : memcpy(planstate->worker_jit_instrument, shared_jit, ibytes);
1120 GIC 12 : }
1121 ECB :
1122 : /*
1123 : * Finish parallel execution. We wait for parallel workers to finish, and
1124 : * accumulate their buffer/WAL usage.
1125 : */
1126 : void
1127 GIC 823 : ExecParallelFinish(ParallelExecutorInfo *pei)
1128 : {
1129 CBC 823 : int nworkers = pei->pcxt->nworkers_launched;
1130 : int i;
1131 ECB :
1132 : /* Make this be a no-op if called twice in a row. */
1133 GIC 823 : if (pei->finished)
1134 374 : return;
1135 ECB :
1136 : /*
1137 : * Detach from tuple queues ASAP, so that any still-active workers will
1138 : * notice that no further results are wanted.
1139 : */
1140 GIC 449 : if (pei->tqueue != NULL)
1141 : {
1142 CBC 1656 : for (i = 0; i < nworkers; i++)
1143 GIC 1207 : shm_mq_detach(pei->tqueue[i]);
1144 CBC 449 : pfree(pei->tqueue);
1145 449 : pei->tqueue = NULL;
1146 ECB : }
1147 :
1148 : /*
1149 : * While we're waiting for the workers to finish, let's get rid of the
1150 : * tuple queue readers. (Any other local cleanup could be done here too.)
1151 : */
1152 GIC 449 : if (pei->reader != NULL)
1153 : {
1154 CBC 1647 : for (i = 0; i < nworkers; i++)
1155 GIC 1207 : DestroyTupleQueueReader(pei->reader[i]);
1156 CBC 440 : pfree(pei->reader);
1157 440 : pei->reader = NULL;
1158 ECB : }
1159 :
1160 : /* Now wait for the workers to finish. */
1161 GIC 449 : WaitForParallelWorkersToFinish(pei->pcxt);
1162 :
1163 ECB : /*
1164 : * Next, accumulate buffer/WAL usage. (This must wait for the workers to
1165 : * finish, or we might get incomplete data.)
1166 : */
1167 GIC 1656 : for (i = 0; i < nworkers; i++)
1168 1207 : InstrAccumParallelQuery(&pei->buffer_usage[i], &pei->wal_usage[i]);
1169 ECB :
1170 CBC 449 : pei->finished = true;
1171 : }
1172 ECB :
1173 : /*
1174 : * Accumulate instrumentation, and then clean up whatever ParallelExecutorInfo
1175 : * resources still exist after ExecParallelFinish. We separate these
1176 : * routines because someone might want to examine the contents of the DSM
1177 : * after ExecParallelFinish and before calling this routine.
1178 : */
1179 : void
1180 GIC 320 : ExecParallelCleanup(ParallelExecutorInfo *pei)
1181 : {
1182 ECB : /* Accumulate instrumentation, if any. */
1183 GIC 320 : if (pei->instrumentation)
1184 93 : ExecParallelRetrieveInstrumentation(pei->planstate,
1185 ECB : pei->instrumentation);
1186 :
1187 : /* Accumulate JIT instrumentation, if any. */
1188 GIC 320 : if (pei->jit_instrumentation)
1189 12 : ExecParallelRetrieveJitInstrumentation(pei->planstate,
1190 CBC 12 : pei->jit_instrumentation);
1191 ECB :
1192 : /* Free any serialized parameters. */
1193 GIC 320 : if (DsaPointerIsValid(pei->param_exec))
1194 : {
1195 CBC 15 : dsa_free(pei->area, pei->param_exec);
1196 GIC 15 : pei->param_exec = InvalidDsaPointer;
1197 ECB : }
1198 CBC 320 : if (pei->area != NULL)
1199 : {
1200 320 : dsa_detach(pei->area);
1201 GIC 320 : pei->area = NULL;
1202 ECB : }
1203 CBC 320 : if (pei->pcxt != NULL)
1204 : {
1205 320 : DestroyParallelContext(pei->pcxt);
1206 GIC 320 : pei->pcxt = NULL;
1207 ECB : }
1208 CBC 320 : pfree(pei);
1209 GIC 320 : }
1210 ECB :
1211 : /*
1212 : * Create a DestReceiver to write tuples we produce to the shm_mq designated
1213 : * for that purpose.
1214 : */
1215 : static DestReceiver *
1216 GIC 1210 : ExecParallelGetReceiver(dsm_segment *seg, shm_toc *toc)
1217 : {
1218 ECB : char *mqspace;
1219 : shm_mq *mq;
1220 :
1221 GIC 1210 : mqspace = shm_toc_lookup(toc, PARALLEL_KEY_TUPLE_QUEUE, false);
1222 1210 : mqspace += ParallelWorkerNumber * PARALLEL_TUPLE_QUEUE_SIZE;
1223 CBC 1210 : mq = (shm_mq *) mqspace;
1224 1210 : shm_mq_set_sender(mq, MyProc);
1225 1210 : return CreateTupleQueueDestReceiver(shm_mq_attach(mq, seg, NULL));
1226 ECB : }
1227 :
1228 : /*
1229 : * Create a QueryDesc for the PlannedStmt we are to execute, and return it.
1230 : */
1231 : static QueryDesc *
1232 GIC 1210 : ExecParallelGetQueryDesc(shm_toc *toc, DestReceiver *receiver,
1233 : int instrument_options)
1234 ECB : {
1235 : char *pstmtspace;
1236 : char *paramspace;
1237 : PlannedStmt *pstmt;
1238 : ParamListInfo paramLI;
1239 : char *queryString;
1240 :
1241 : /* Get the query string from shared memory */
1242 GIC 1210 : queryString = shm_toc_lookup(toc, PARALLEL_KEY_QUERY_TEXT, false);
1243 :
1244 ECB : /* Reconstruct leader-supplied PlannedStmt. */
1245 GIC 1210 : pstmtspace = shm_toc_lookup(toc, PARALLEL_KEY_PLANNEDSTMT, false);
1246 1210 : pstmt = (PlannedStmt *) stringToNode(pstmtspace);
1247 ECB :
1248 : /* Reconstruct ParamListInfo. */
1249 GIC 1210 : paramspace = shm_toc_lookup(toc, PARALLEL_KEY_PARAMLISTINFO, false);
1250 1210 : paramLI = RestoreParamList(¶mspace);
1251 ECB :
1252 : /* Create a QueryDesc for the query. */
1253 GIC 1210 : return CreateQueryDesc(pstmt,
1254 : queryString,
1255 ECB : GetActiveSnapshot(), InvalidSnapshot,
1256 : receiver, paramLI, NULL, instrument_options);
1257 : }
1258 :
1259 : /*
1260 : * Copy instrumentation information from this node and its descendants into
1261 : * dynamic shared memory, so that the parallel leader can retrieve it.
1262 : */
1263 : static bool
1264 GIC 1169 : ExecParallelReportInstrumentation(PlanState *planstate,
1265 : SharedExecutorInstrumentation *instrumentation)
1266 ECB : {
1267 : int i;
1268 GIC 1169 : int plan_node_id = planstate->plan->plan_node_id;
1269 : Instrumentation *instrument;
1270 ECB :
1271 GIC 1169 : InstrEndLoop(planstate->instrument);
1272 :
1273 ECB : /*
1274 : * If we shuffled the plan_node_id values in ps_instrument into sorted
1275 : * order, we could use binary search here. This might matter someday if
1276 : * we're pushing down sufficiently large plan trees. For now, do it the
1277 : * slow, dumb way.
1278 : */
1279 GIC 3713 : for (i = 0; i < instrumentation->num_plan_nodes; ++i)
1280 3713 : if (instrumentation->plan_node_id[i] == plan_node_id)
1281 CBC 1169 : break;
1282 1169 : if (i >= instrumentation->num_plan_nodes)
1283 LBC 0 : elog(ERROR, "plan node %d not found", plan_node_id);
1284 ECB :
1285 EUB : /*
1286 : * Add our statistics to the per-node, per-worker totals. It's possible
1287 : * that this could happen more than once if we relaunched workers.
1288 : */
1289 GIC 1169 : instrument = GetInstrumentationArray(instrumentation);
1290 1169 : instrument += i * instrumentation->num_workers;
1291 CBC 1169 : Assert(IsParallelWorker());
1292 1169 : Assert(ParallelWorkerNumber < instrumentation->num_workers);
1293 1169 : InstrAggNode(&instrument[ParallelWorkerNumber], planstate->instrument);
1294 ECB :
1295 CBC 1169 : return planstate_tree_walker(planstate, ExecParallelReportInstrumentation,
1296 : instrumentation);
1297 ECB : }
1298 :
1299 : /*
1300 : * Initialize the PlanState and its descendants with the information
1301 : * retrieved from shared memory. This has to be done once the PlanState
1302 : * is allocated and initialized by executor; that is, after ExecutorStart().
1303 : */
1304 : static bool
1305 GIC 3962 : ExecParallelInitializeWorker(PlanState *planstate, ParallelWorkerContext *pwcxt)
1306 : {
1307 CBC 3962 : if (planstate == NULL)
1308 UIC 0 : return false;
1309 ECB :
1310 GBC 3962 : switch (nodeTag(planstate))
1311 : {
1312 CBC 1626 : case T_SeqScanState:
1313 GIC 1626 : if (planstate->plan->parallel_aware)
1314 CBC 1312 : ExecSeqScanInitializeWorker((SeqScanState *) planstate, pwcxt);
1315 1626 : break;
1316 186 : case T_IndexScanState:
1317 186 : if (planstate->plan->parallel_aware)
1318 48 : ExecIndexScanInitializeWorker((IndexScanState *) planstate,
1319 ECB : pwcxt);
1320 CBC 186 : break;
1321 GIC 118 : case T_IndexOnlyScanState:
1322 CBC 118 : if (planstate->plan->parallel_aware)
1323 100 : ExecIndexOnlyScanInitializeWorker((IndexOnlyScanState *) planstate,
1324 ECB : pwcxt);
1325 CBC 118 : break;
1326 UIC 0 : case T_ForeignScanState:
1327 LBC 0 : if (planstate->plan->parallel_aware)
1328 UBC 0 : ExecForeignScanInitializeWorker((ForeignScanState *) planstate,
1329 EUB : pwcxt);
1330 UBC 0 : break;
1331 GIC 182 : case T_AppendState:
1332 GBC 182 : if (planstate->plan->parallel_aware)
1333 CBC 152 : ExecAppendInitializeWorker((AppendState *) planstate, pwcxt);
1334 182 : break;
1335 LBC 0 : case T_CustomScanState:
1336 0 : if (planstate->plan->parallel_aware)
1337 UBC 0 : ExecCustomScanInitializeWorker((CustomScanState *) planstate,
1338 EUB : pwcxt);
1339 UBC 0 : break;
1340 GIC 139 : case T_BitmapHeapScanState:
1341 GBC 139 : if (planstate->plan->parallel_aware)
1342 CBC 138 : ExecBitmapHeapInitializeWorker((BitmapHeapScanState *) planstate,
1343 ECB : pwcxt);
1344 CBC 139 : break;
1345 GIC 267 : case T_HashJoinState:
1346 CBC 267 : if (planstate->plan->parallel_aware)
1347 147 : ExecHashJoinInitializeWorker((HashJoinState *) planstate,
1348 ECB : pwcxt);
1349 CBC 267 : break;
1350 GIC 267 : case T_HashState:
1351 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
1352 CBC 267 : ExecHashInitializeWorker((HashState *) planstate, pwcxt);
1353 GIC 267 : break;
1354 CBC 214 : case T_SortState:
1355 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
1356 CBC 214 : ExecSortInitializeWorker((SortState *) planstate, pwcxt);
1357 GIC 214 : break;
1358 LBC 0 : case T_IncrementalSortState:
1359 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
1360 UBC 0 : ExecIncrementalSortInitializeWorker((IncrementalSortState *) planstate,
1361 : pwcxt);
1362 0 : break;
1363 GIC 768 : case T_AggState:
1364 EUB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
1365 CBC 768 : ExecAggInitializeWorker((AggState *) planstate, pwcxt);
1366 GIC 768 : break;
1367 CBC 6 : case T_MemoizeState:
1368 ECB : /* even when not parallel-aware, for EXPLAIN ANALYZE */
1369 CBC 6 : ExecMemoizeInitializeWorker((MemoizeState *) planstate, pwcxt);
1370 GIC 6 : break;
1371 CBC 189 : default:
1372 189 : break;
1373 ECB : }
1374 :
1375 GIC 3962 : return planstate_tree_walker(planstate, ExecParallelInitializeWorker,
1376 : pwcxt);
1377 ECB : }
1378 :
1379 : /*
1380 : * Main entrypoint for parallel query worker processes.
1381 : *
1382 : * We reach this function from ParallelWorkerMain, so the setup necessary to
1383 : * create a sensible parallel environment has already been done;
1384 : * ParallelWorkerMain worries about stuff like the transaction state, combo
1385 : * CID mappings, and GUC values, so we don't need to deal with any of that
1386 : * here.
1387 : *
1388 : * Our job is to deal with concerns specific to the executor. The parallel
1389 : * group leader will have stored a serialized PlannedStmt, and it's our job
1390 : * to execute that plan and write the resulting tuples to the appropriate
1391 : * tuple queue. Various bits of supporting information that we need in order
1392 : * to do this are also stored in the dsm_segment and can be accessed through
1393 : * the shm_toc.
1394 : */
1395 : void
1396 GIC 1210 : ParallelQueryMain(dsm_segment *seg, shm_toc *toc)
1397 : {
1398 ECB : FixedParallelExecutorState *fpes;
1399 : BufferUsage *buffer_usage;
1400 : WalUsage *wal_usage;
1401 : DestReceiver *receiver;
1402 : QueryDesc *queryDesc;
1403 : SharedExecutorInstrumentation *instrumentation;
1404 : SharedJitInstrumentation *jit_instrumentation;
1405 GIC 1210 : int instrument_options = 0;
1406 : void *area_space;
1407 ECB : dsa_area *area;
1408 : ParallelWorkerContext pwcxt;
1409 :
1410 : /* Get fixed-size state. */
1411 GIC 1210 : fpes = shm_toc_lookup(toc, PARALLEL_KEY_EXECUTOR_FIXED, false);
1412 :
1413 ECB : /* Set up DestReceiver, SharedExecutorInstrumentation, and QueryDesc. */
1414 GIC 1210 : receiver = ExecParallelGetReceiver(seg, toc);
1415 1210 : instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_INSTRUMENTATION, true);
1416 CBC 1210 : if (instrumentation != NULL)
1417 368 : instrument_options = instrumentation->instrument_options;
1418 1210 : jit_instrumentation = shm_toc_lookup(toc, PARALLEL_KEY_JIT_INSTRUMENTATION,
1419 ECB : true);
1420 CBC 1210 : queryDesc = ExecParallelGetQueryDesc(toc, receiver, instrument_options);
1421 :
1422 ECB : /* Setting debug_query_string for individual workers */
1423 GIC 1210 : debug_query_string = queryDesc->sourceText;
1424 :
1425 ECB : /* Report workers' query for monitoring purposes */
1426 GIC 1210 : pgstat_report_activity(STATE_RUNNING, debug_query_string);
1427 :
1428 ECB : /* Attach to the dynamic shared memory area. */
1429 GIC 1210 : area_space = shm_toc_lookup(toc, PARALLEL_KEY_DSA, false);
1430 1210 : area = dsa_attach_in_place(area_space, seg);
1431 ECB :
1432 : /* Start up the executor */
1433 GIC 1210 : queryDesc->plannedstmt->jitFlags = fpes->jit_flags;
1434 1210 : ExecutorStart(queryDesc, fpes->eflags);
1435 ECB :
1436 : /* Special executor initialization steps for parallel workers */
1437 GIC 1210 : queryDesc->planstate->state->es_query_dsa = area;
1438 1210 : if (DsaPointerIsValid(fpes->param_exec))
1439 ECB : {
1440 : char *paramexec_space;
1441 :
1442 GIC 35 : paramexec_space = dsa_get_address(area, fpes->param_exec);
1443 35 : RestoreParamExecParams(paramexec_space, queryDesc->estate);
1444 ECB : }
1445 CBC 1210 : pwcxt.toc = toc;
1446 GIC 1210 : pwcxt.seg = seg;
1447 CBC 1210 : ExecParallelInitializeWorker(queryDesc->planstate, &pwcxt);
1448 ECB :
1449 : /* Pass down any tuple bound */
1450 GIC 1210 : ExecSetTupleBound(fpes->tuples_needed, queryDesc->planstate);
1451 :
1452 ECB : /*
1453 : * Prepare to track buffer/WAL usage during query execution.
1454 : *
1455 : * We do this after starting up the executor to match what happens in the
1456 : * leader, which also doesn't count buffer accesses and WAL activity that
1457 : * occur during executor startup.
1458 : */
1459 GIC 1210 : InstrStartParallelQuery();
1460 :
1461 ECB : /*
1462 : * Run the plan. If we specified a tuple bound, be careful not to demand
1463 : * more tuples than that.
1464 : */
1465 GIC 1210 : ExecutorRun(queryDesc,
1466 : ForwardScanDirection,
1467 CBC 1210 : fpes->tuples_needed < 0 ? (int64) 0 : fpes->tuples_needed,
1468 : true);
1469 ECB :
1470 : /* Shut down the executor */
1471 GIC 1207 : ExecutorFinish(queryDesc);
1472 :
1473 ECB : /* Report buffer/WAL usage during parallel execution. */
1474 GIC 1207 : buffer_usage = shm_toc_lookup(toc, PARALLEL_KEY_BUFFER_USAGE, false);
1475 1207 : wal_usage = shm_toc_lookup(toc, PARALLEL_KEY_WAL_USAGE, false);
1476 CBC 1207 : InstrEndParallelQuery(&buffer_usage[ParallelWorkerNumber],
1477 1207 : &wal_usage[ParallelWorkerNumber]);
1478 ECB :
1479 : /* Report instrumentation data if any instrumentation options are set. */
1480 GIC 1207 : if (instrumentation != NULL)
1481 368 : ExecParallelReportInstrumentation(queryDesc->planstate,
1482 ECB : instrumentation);
1483 :
1484 : /* Report JIT instrumentation data if any */
1485 GIC 1207 : if (queryDesc->estate->es_jit && jit_instrumentation != NULL)
1486 : {
1487 CBC 72 : Assert(ParallelWorkerNumber < jit_instrumentation->num_workers);
1488 GIC 72 : jit_instrumentation->jit_instr[ParallelWorkerNumber] =
1489 CBC 72 : queryDesc->estate->es_jit->instr;
1490 ECB : }
1491 :
1492 : /* Must do this after capturing instrumentation. */
1493 GIC 1207 : ExecutorEnd(queryDesc);
1494 :
1495 ECB : /* Cleanup. */
1496 GIC 1207 : dsa_detach(area);
1497 1207 : FreeQueryDesc(queryDesc);
1498 CBC 1207 : receiver->rDestroy(receiver);
1499 1207 : }
|
