TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * generation.c
4 : * Generational allocator definitions.
5 : *
6 : * Generation is a custom MemoryContext implementation designed for cases of
7 : * chunks with similar lifespan.
8 : *
9 : * Portions Copyright (c) 2017-2023, PostgreSQL Global Development Group
10 : *
11 : * IDENTIFICATION
12 : * src/backend/utils/mmgr/generation.c
13 : *
14 : *
15 : * This memory context is based on the assumption that the chunks are freed
16 : * roughly in the same order as they were allocated (FIFO), or in groups with
17 : * similar lifespan (generations - hence the name of the context). This is
18 : * typical for various queue-like use cases, i.e. when tuples are constructed,
19 : * processed and then thrown away.
20 : *
21 : * The memory context uses a very simple approach to free space management.
22 : * Instead of a complex global freelist, each block tracks a number
23 : * of allocated and freed chunks. The block is classed as empty when the
24 : * number of free chunks is equal to the number of allocated chunks. When
25 : * this occurs, instead of freeing the block, we try to "recycle" it, i.e.
26 : * reuse it for new allocations. This is done by setting the block in the
27 : * context's 'freeblock' field. If the freeblock field is already occupied
28 : * by another free block we simply return the newly empty block to malloc.
29 : *
30 : * This approach to free blocks requires fewer malloc/free calls for truly
31 : * first allocated, first free'd allocation patterns.
32 : *
33 : *-------------------------------------------------------------------------
34 : */
35 :
36 : #include "postgres.h"
37 :
38 : #include "lib/ilist.h"
39 : #include "port/pg_bitutils.h"
40 : #include "utils/memdebug.h"
41 : #include "utils/memutils.h"
42 : #include "utils/memutils_memorychunk.h"
43 : #include "utils/memutils_internal.h"
44 :
45 :
46 : #define Generation_BLOCKHDRSZ MAXALIGN(sizeof(GenerationBlock))
47 : #define Generation_CHUNKHDRSZ sizeof(MemoryChunk)
48 :
49 : #define Generation_CHUNK_FRACTION 8
50 :
51 : typedef struct GenerationBlock GenerationBlock; /* forward reference */
52 :
53 : typedef void *GenerationPointer;
54 :
55 : /*
56 : * GenerationContext is a simple memory context not reusing allocated chunks,
57 : * and freeing blocks once all chunks are freed.
58 : */
59 : typedef struct GenerationContext
60 : {
61 : MemoryContextData header; /* Standard memory-context fields */
62 :
63 : /* Generational context parameters */
64 : Size initBlockSize; /* initial block size */
65 : Size maxBlockSize; /* maximum block size */
66 : Size nextBlockSize; /* next block size to allocate */
67 : Size allocChunkLimit; /* effective chunk size limit */
68 :
69 : GenerationBlock *block; /* current (most recently allocated) block, or
70 : * NULL if we've just freed the most recent
71 : * block */
72 : GenerationBlock *freeblock; /* pointer to a block that's being recycled,
73 : * or NULL if there's no such block. */
74 : GenerationBlock *keeper; /* keep this block over resets */
75 : dlist_head blocks; /* list of blocks */
76 : } GenerationContext;
77 :
78 : /*
79 : * GenerationBlock
80 : * GenerationBlock is the unit of memory that is obtained by generation.c
81 : * from malloc(). It contains zero or more MemoryChunks, which are the
82 : * units requested by palloc() and freed by pfree(). MemoryChunks cannot
83 : * be returned to malloc() individually, instead pfree() updates the free
84 : * counter of the block and when all chunks in a block are free the whole
85 : * block can be returned to malloc().
86 : *
87 : * GenerationBlock is the header data for a block --- the usable space
88 : * within the block begins at the next alignment boundary.
89 : */
90 : struct GenerationBlock
91 : {
92 : dlist_node node; /* doubly-linked list of blocks */
93 : GenerationContext *context; /* pointer back to the owning context */
94 : Size blksize; /* allocated size of this block */
95 : int nchunks; /* number of chunks in the block */
96 : int nfree; /* number of free chunks */
97 : char *freeptr; /* start of free space in this block */
98 : char *endptr; /* end of space in this block */
99 : };
100 :
101 : /*
102 : * Only the "hdrmask" field should be accessed outside this module.
103 : * We keep the rest of an allocated chunk's header marked NOACCESS when using
104 : * valgrind. But note that freed chunk headers are kept accessible, for
105 : * simplicity.
106 : */
107 : #define GENERATIONCHUNK_PRIVATE_LEN offsetof(MemoryChunk, hdrmask)
108 :
109 : /*
110 : * GenerationIsValid
111 : * True iff set is valid generation set.
112 : */
113 : #define GenerationIsValid(set) \
114 : (PointerIsValid(set) && IsA(set, GenerationContext))
115 :
116 : /*
117 : * GenerationBlockIsValid
118 : * True iff block is valid block of generation set.
119 : */
120 : #define GenerationBlockIsValid(block) \
121 : (PointerIsValid(block) && GenerationIsValid((block)->context))
122 :
123 : /*
124 : * We always store external chunks on a dedicated block. This makes fetching
125 : * the block from an external chunk easy since it's always the first and only
126 : * chunk on the block.
127 : */
128 : #define ExternalChunkGetBlock(chunk) \
129 : (GenerationBlock *) ((char *) chunk - Generation_BLOCKHDRSZ)
130 :
131 : /* Inlined helper functions */
132 : static inline void GenerationBlockInit(GenerationContext *context,
133 : GenerationBlock *block,
134 : Size blksize);
135 : static inline bool GenerationBlockIsEmpty(GenerationBlock *block);
136 : static inline void GenerationBlockMarkEmpty(GenerationBlock *block);
137 ECB : static inline Size GenerationBlockFreeBytes(GenerationBlock *block);
138 : static inline void GenerationBlockFree(GenerationContext *set,
139 : GenerationBlock *block);
140 :
141 :
142 : /*
143 : * Public routines
144 : */
145 :
146 :
147 : /*
148 : * GenerationContextCreate
149 : * Create a new Generation context.
150 : *
151 : * parent: parent context, or NULL if top-level context
152 : * name: name of context (must be statically allocated)
153 EUB : * minContextSize: minimum context size
154 : * initBlockSize: initial allocation block size
155 : * maxBlockSize: maximum allocation block size
156 : */
157 : MemoryContext
158 GIC 190972 : GenerationContextCreate(MemoryContext parent,
159 : const char *name,
160 : Size minContextSize,
161 : Size initBlockSize,
162 : Size maxBlockSize)
163 : {
164 : Size firstBlockSize;
165 ECB : Size allocSize;
166 : GenerationContext *set;
167 : GenerationBlock *block;
168 :
169 : /* ensure MemoryChunk's size is properly maxaligned */
170 : StaticAssertDecl(Generation_CHUNKHDRSZ == MAXALIGN(Generation_CHUNKHDRSZ),
171 : "sizeof(MemoryChunk) is not maxaligned");
172 :
173 : /*
174 : * First, validate allocation parameters. Asserts seem sufficient because
175 : * nobody varies their parameters at runtime. We somewhat arbitrarily
176 : * enforce a minimum 1K block size. We restrict the maximum block size to
177 : * MEMORYCHUNK_MAX_BLOCKOFFSET as MemoryChunks are limited to this in
178 : * regards to addressing the offset between the chunk and the block that
179 : * the chunk is stored on. We would be unable to store the offset between
180 : * the chunk and block for any chunks that were beyond
181 : * MEMORYCHUNK_MAX_BLOCKOFFSET bytes into the block if the block was to be
182 : * larger than this.
183 : */
184 GIC 190972 : Assert(initBlockSize == MAXALIGN(initBlockSize) &&
185 : initBlockSize >= 1024);
186 CBC 190972 : Assert(maxBlockSize == MAXALIGN(maxBlockSize) &&
187 : maxBlockSize >= initBlockSize &&
188 : AllocHugeSizeIsValid(maxBlockSize)); /* must be safe to double */
189 190972 : Assert(minContextSize == 0 ||
190 ECB : (minContextSize == MAXALIGN(minContextSize) &&
191 : minContextSize >= 1024 &&
192 : minContextSize <= maxBlockSize));
193 GNC 190972 : Assert(maxBlockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET);
194 :
195 : /* Determine size of initial block */
196 GIC 190972 : allocSize = MAXALIGN(sizeof(GenerationContext)) +
197 : Generation_BLOCKHDRSZ + Generation_CHUNKHDRSZ;
198 190972 : if (minContextSize != 0)
199 865 : allocSize = Max(allocSize, minContextSize);
200 : else
201 190107 : allocSize = Max(allocSize, initBlockSize);
202 :
203 : /*
204 ECB : * Allocate the initial block. Unlike other generation.c blocks, it
205 : * starts with the context header and its block header follows that.
206 : */
207 CBC 190972 : set = (GenerationContext *) malloc(allocSize);
208 GIC 190972 : if (set == NULL)
209 : {
210 LBC 0 : MemoryContextStats(TopMemoryContext);
211 UIC 0 : ereport(ERROR,
212 : (errcode(ERRCODE_OUT_OF_MEMORY),
213 : errmsg("out of memory"),
214 : errdetail("Failed while creating memory context \"%s\".",
215 : name)));
216 ECB : }
217 :
218 : /*
219 : * Avoid writing code that can fail between here and MemoryContextCreate;
220 : * we'd leak the header if we ereport in this stretch.
221 : */
222 GIC 190972 : dlist_init(&set->blocks);
223 :
224 : /* Fill in the initial block's block header */
225 190972 : block = (GenerationBlock *) (((char *) set) + MAXALIGN(sizeof(GenerationContext)));
226 : /* determine the block size and initialize it */
227 190972 : firstBlockSize = allocSize - MAXALIGN(sizeof(GenerationContext));
228 GNC 190972 : GenerationBlockInit(set, block, firstBlockSize);
229 ECB :
230 : /* add it to the doubly-linked list of blocks */
231 CBC 190972 : dlist_push_head(&set->blocks, &block->node);
232 :
233 : /* use it as the current allocation block */
234 190972 : set->block = block;
235 :
236 : /* No free block, yet */
237 GIC 190972 : set->freeblock = NULL;
238 ECB :
239 : /* Mark block as not to be released at reset time */
240 GIC 190972 : set->keeper = block;
241 :
242 : /* Fill in GenerationContext-specific header fields */
243 190972 : set->initBlockSize = initBlockSize;
244 190972 : set->maxBlockSize = maxBlockSize;
245 190972 : set->nextBlockSize = initBlockSize;
246 ECB :
247 : /*
248 : * Compute the allocation chunk size limit for this context.
249 : *
250 : * Limit the maximum size a non-dedicated chunk can be so that we can fit
251 : * at least Generation_CHUNK_FRACTION of chunks this big onto the maximum
252 : * sized block. We must further limit this value so that it's no more
253 : * than MEMORYCHUNK_MAX_VALUE. We're unable to have non-external chunks
254 : * larger than that value as we store the chunk size in the MemoryChunk
255 : * 'value' field in the call to MemoryChunkSetHdrMask().
256 : */
257 GNC 190972 : set->allocChunkLimit = Min(maxBlockSize, MEMORYCHUNK_MAX_VALUE);
258 CBC 190972 : while ((Size) (set->allocChunkLimit + Generation_CHUNKHDRSZ) >
259 GIC 954860 : (Size) ((Size) (maxBlockSize - Generation_BLOCKHDRSZ) / Generation_CHUNK_FRACTION))
260 CBC 763888 : set->allocChunkLimit >>= 1;
261 :
262 : /* Finally, do the type-independent part of context creation */
263 GIC 190972 : MemoryContextCreate((MemoryContext) set,
264 ECB : T_GenerationContext,
265 : MCTX_GENERATION_ID,
266 : parent,
267 : name);
268 :
269 GIC 190972 : ((MemoryContext) set)->mem_allocated = firstBlockSize;
270 ECB :
271 CBC 190972 : return (MemoryContext) set;
272 ECB : }
273 :
274 : /*
275 : * GenerationReset
276 : * Frees all memory which is allocated in the given set.
277 : *
278 : * The code simply frees all the blocks in the context - we don't keep any
279 : * keeper blocks or anything like that.
280 : */
281 : void
282 CBC 192196 : GenerationReset(MemoryContext context)
283 : {
284 192196 : GenerationContext *set = (GenerationContext *) context;
285 ECB : dlist_mutable_iter miter;
286 :
287 GNC 192196 : Assert(GenerationIsValid(set));
288 :
289 : #ifdef MEMORY_CONTEXT_CHECKING
290 : /* Check for corruption and leaks before freeing */
291 GIC 192196 : GenerationCheck(context);
292 : #endif
293 :
294 : /*
295 : * NULLify the free block pointer. We must do this before calling
296 : * GenerationBlockFree as that function never expects to free the
297 : * freeblock.
298 : */
299 192196 : set->freeblock = NULL;
300 :
301 CBC 409807 : dlist_foreach_modify(miter, &set->blocks)
302 : {
303 217611 : GenerationBlock *block = dlist_container(GenerationBlock, node, miter.cur);
304 :
305 GIC 217611 : if (block == set->keeper)
306 192196 : GenerationBlockMarkEmpty(block);
307 : else
308 25415 : GenerationBlockFree(set, block);
309 ECB : }
310 :
311 : /* set it so new allocations to make use of the keeper block */
312 GIC 192196 : set->block = set->keeper;
313 ECB :
314 : /* Reset block size allocation sequence, too */
315 GIC 192196 : set->nextBlockSize = set->initBlockSize;
316 :
317 ECB : /* Ensure there is only 1 item in the dlist */
318 GIC 192196 : Assert(!dlist_is_empty(&set->blocks));
319 192196 : Assert(!dlist_has_next(&set->blocks, dlist_head_node(&set->blocks)));
320 CBC 192196 : }
321 :
322 ECB : /*
323 : * GenerationDelete
324 : * Free all memory which is allocated in the given context.
325 : */
326 : void
327 GIC 190844 : GenerationDelete(MemoryContext context)
328 ECB : {
329 : /* Reset to release all releasable GenerationBlocks */
330 GIC 190844 : GenerationReset(context);
331 ECB : /* And free the context header and keeper block */
332 CBC 190844 : free(context);
333 190844 : }
334 ECB :
335 : /*
336 : * GenerationAlloc
337 : * Returns pointer to allocated memory of given size or NULL if
338 : * request could not be completed; memory is added to the set.
339 : *
340 : * No request may exceed:
341 : * MAXALIGN_DOWN(SIZE_MAX) - Generation_BLOCKHDRSZ - Generation_CHUNKHDRSZ
342 : * All callers use a much-lower limit.
343 : *
344 : * Note: when using valgrind, it doesn't matter how the returned allocation
345 : * is marked, as mcxt.c will set it to UNDEFINED. In some paths we will
346 : * return space that is marked NOACCESS - GenerationRealloc has to beware!
347 : */
348 : void *
349 GIC 18301321 : GenerationAlloc(MemoryContext context, Size size)
350 : {
351 18301321 : GenerationContext *set = (GenerationContext *) context;
352 : GenerationBlock *block;
353 : MemoryChunk *chunk;
354 : Size chunk_size;
355 : Size required_size;
356 :
357 GNC 18301321 : Assert(GenerationIsValid(set));
358 :
359 : #ifdef MEMORY_CONTEXT_CHECKING
360 : /* ensure there's always space for the sentinel byte */
361 18301321 : chunk_size = MAXALIGN(size + 1);
362 : #else
363 : chunk_size = MAXALIGN(size);
364 : #endif
365 18301321 : required_size = chunk_size + Generation_CHUNKHDRSZ;
366 ECB :
367 : /* is it an over-sized chunk? if yes, allocate special block */
368 GIC 18301321 : if (chunk_size > set->allocChunkLimit)
369 : {
370 84 : Size blksize = required_size + Generation_BLOCKHDRSZ;
371 :
372 84 : block = (GenerationBlock *) malloc(blksize);
373 84 : if (block == NULL)
374 UIC 0 : return NULL;
375 ECB :
376 GIC 84 : context->mem_allocated += blksize;
377 :
378 : /* block with a single (used) chunk */
379 GNC 84 : block->context = set;
380 GIC 84 : block->blksize = blksize;
381 84 : block->nchunks = 1;
382 84 : block->nfree = 0;
383 :
384 : /* the block is completely full */
385 84 : block->freeptr = block->endptr = ((char *) block) + blksize;
386 :
387 GNC 84 : chunk = (MemoryChunk *) (((char *) block) + Generation_BLOCKHDRSZ);
388 :
389 : /* mark the MemoryChunk as externally managed */
390 84 : MemoryChunkSetHdrMaskExternal(chunk, MCTX_GENERATION_ID);
391 :
392 : #ifdef MEMORY_CONTEXT_CHECKING
393 GIC 84 : chunk->requested_size = size;
394 ECB : /* set mark to catch clobber of "unused" space */
395 GNC 84 : Assert(size < chunk_size);
396 84 : set_sentinel(MemoryChunkGetPointer(chunk), size);
397 ECB : #endif
398 : #ifdef RANDOMIZE_ALLOCATED_MEMORY
399 : /* fill the allocated space with junk */
400 : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
401 : #endif
402 :
403 EUB : /* add the block to the list of allocated blocks */
404 GBC 84 : dlist_push_head(&set->blocks, &block->node);
405 :
406 EUB : /* Ensure any padding bytes are marked NOACCESS. */
407 : VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
408 : chunk_size - size);
409 :
410 : /* Disallow external access to private part of chunk header. */
411 : VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN);
412 :
413 GNC 84 : return MemoryChunkGetPointer(chunk);
414 ECB : }
415 EUB :
416 : /*
417 : * Not an oversized chunk. We try to first make use of the current block,
418 : * but if there's not enough space in it, instead of allocating a new
419 : * block, we look to see if the freeblock is empty and has enough space.
420 : * If not, we'll also try the same using the keeper block. The keeper
421 : * block may have become empty and we have no other way to reuse it again
422 : * if we don't try to use it explicitly here.
423 : *
424 : * We don't want to start filling the freeblock before the current block
425 ECB : * is full, otherwise we may cause fragmentation in FIFO type workloads.
426 : * We only switch to using the freeblock or keeper block if those blocks
427 : * are completely empty. If we didn't do that we could end up fragmenting
428 : * consecutive allocations over multiple blocks which would be a problem
429 : * that would compound over time.
430 : */
431 CBC 18301237 : block = set->block;
432 :
433 GIC 36602474 : if (block == NULL ||
434 CBC 18301237 : GenerationBlockFreeBytes(block) < required_size)
435 EUB : {
436 : Size blksize;
437 CBC 25331 : GenerationBlock *freeblock = set->freeblock;
438 :
439 25331 : if (freeblock != NULL &&
440 UBC 0 : GenerationBlockIsEmpty(freeblock) &&
441 UIC 0 : GenerationBlockFreeBytes(freeblock) >= required_size)
442 ECB : {
443 UIC 0 : block = freeblock;
444 :
445 ECB : /*
446 : * Zero out the freeblock as we'll set this to the current block
447 : * below
448 : */
449 UIC 0 : set->freeblock = NULL;
450 : }
451 CBC 25331 : else if (GenerationBlockIsEmpty(set->keeper) &&
452 UIC 0 : GenerationBlockFreeBytes(set->keeper) >= required_size)
453 : {
454 0 : block = set->keeper;
455 ECB : }
456 : else
457 : {
458 : /*
459 : * The first such block has size initBlockSize, and we double the
460 : * space in each succeeding block, but not more than maxBlockSize.
461 : */
462 CBC 25331 : blksize = set->nextBlockSize;
463 GIC 25331 : set->nextBlockSize <<= 1;
464 25331 : if (set->nextBlockSize > set->maxBlockSize)
465 4 : set->nextBlockSize = set->maxBlockSize;
466 :
467 ECB : /* we'll need a block hdr too, so add that to the required size */
468 CBC 25331 : required_size += Generation_BLOCKHDRSZ;
469 :
470 ECB : /* round the size up to the next power of 2 */
471 GIC 25331 : if (blksize < required_size)
472 LBC 0 : blksize = pg_nextpower2_size_t(required_size);
473 :
474 CBC 25331 : block = (GenerationBlock *) malloc(blksize);
475 :
476 25331 : if (block == NULL)
477 LBC 0 : return NULL;
478 :
479 GIC 25331 : context->mem_allocated += blksize;
480 :
481 : /* initialize the new block */
482 GNC 25331 : GenerationBlockInit(set, block, blksize);
483 :
484 : /* add it to the doubly-linked list of blocks */
485 GIC 25331 : dlist_push_head(&set->blocks, &block->node);
486 :
487 : /* Zero out the freeblock in case it's become full */
488 25331 : set->freeblock = NULL;
489 : }
490 :
491 ECB : /* and also use it as the current allocation block */
492 GIC 25331 : set->block = block;
493 : }
494 :
495 : /* we're supposed to have a block with enough free space now */
496 18301237 : Assert(block != NULL);
497 18301237 : Assert((block->endptr - block->freeptr) >= Generation_CHUNKHDRSZ + chunk_size);
498 :
499 GNC 18301237 : chunk = (MemoryChunk *) block->freeptr;
500 ECB :
501 : /* Prepare to initialize the chunk header. */
502 : VALGRIND_MAKE_MEM_UNDEFINED(chunk, Generation_CHUNKHDRSZ);
503 :
504 CBC 18301237 : block->nchunks += 1;
505 18301237 : block->freeptr += (Generation_CHUNKHDRSZ + chunk_size);
506 ECB :
507 GIC 18301237 : Assert(block->freeptr <= block->endptr);
508 ECB :
509 GNC 18301237 : MemoryChunkSetHdrMask(chunk, block, chunk_size, MCTX_GENERATION_ID);
510 : #ifdef MEMORY_CONTEXT_CHECKING
511 CBC 18301237 : chunk->requested_size = size;
512 : /* set mark to catch clobber of "unused" space */
513 GNC 18301237 : Assert(size < chunk_size);
514 18301237 : set_sentinel(MemoryChunkGetPointer(chunk), size);
515 : #endif
516 : #ifdef RANDOMIZE_ALLOCATED_MEMORY
517 : /* fill the allocated space with junk */
518 : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
519 : #endif
520 ECB :
521 : /* Ensure any padding bytes are marked NOACCESS. */
522 : VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
523 : chunk_size - size);
524 :
525 : /* Disallow external access to private part of chunk header. */
526 : VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN);
527 :
528 GNC 18301237 : return MemoryChunkGetPointer(chunk);
529 : }
530 :
531 ECB : /*
532 : * GenerationBlockInit
533 : * Initializes 'block' assuming 'blksize'. Does not update the context's
534 : * mem_allocated field.
535 : */
536 : static inline void
537 GNC 216303 : GenerationBlockInit(GenerationContext *context, GenerationBlock *block,
538 : Size blksize)
539 : {
540 216303 : block->context = context;
541 GIC 216303 : block->blksize = blksize;
542 216303 : block->nchunks = 0;
543 216303 : block->nfree = 0;
544 ECB :
545 CBC 216303 : block->freeptr = ((char *) block) + Generation_BLOCKHDRSZ;
546 216303 : block->endptr = ((char *) block) + blksize;
547 ECB :
548 : /* Mark unallocated space NOACCESS. */
549 : VALGRIND_MAKE_MEM_NOACCESS(block->freeptr,
550 : blksize - Generation_BLOCKHDRSZ);
551 GIC 216303 : }
552 :
553 : /*
554 ECB : * GenerationBlockIsEmpty
555 : * Returns true iif 'block' contains no chunks
556 : */
557 : static inline bool
558 GIC 25331 : GenerationBlockIsEmpty(GenerationBlock *block)
559 : {
560 25331 : return (block->nchunks == 0);
561 : }
562 :
563 : /*
564 ECB : * GenerationBlockMarkEmpty
565 : * Set a block as empty. Does not free the block.
566 : */
567 : static inline void
568 GIC 197417 : GenerationBlockMarkEmpty(GenerationBlock *block)
569 ECB : {
570 : #if defined(USE_VALGRIND) || defined(CLOBBER_FREED_MEMORY)
571 GIC 197417 : char *datastart = ((char *) block) + Generation_BLOCKHDRSZ;
572 ECB : #endif
573 :
574 : #ifdef CLOBBER_FREED_MEMORY
575 GIC 197417 : wipe_mem(datastart, block->freeptr - datastart);
576 : #else
577 ECB : /* wipe_mem() would have done this */
578 : VALGRIND_MAKE_MEM_NOACCESS(datastart, block->freeptr - datastart);
579 : #endif
580 :
581 : /* Reset the block, but don't return it to malloc */
582 GIC 197417 : block->nchunks = 0;
583 197417 : block->nfree = 0;
584 197417 : block->freeptr = ((char *) block) + Generation_BLOCKHDRSZ;
585 197417 : }
586 :
587 : /*
588 : * GenerationBlockFreeBytes
589 ECB : * Returns the number of bytes free in 'block'
590 : */
591 : static inline Size
592 GIC 18301237 : GenerationBlockFreeBytes(GenerationBlock *block)
593 : {
594 18301237 : return (block->endptr - block->freeptr);
595 : }
596 :
597 : /*
598 : * GenerationBlockFree
599 ECB : * Remove 'block' from 'set' and release the memory consumed by it.
600 : */
601 EUB : static inline void
602 GIC 25415 : GenerationBlockFree(GenerationContext *set, GenerationBlock *block)
603 : {
604 : /* Make sure nobody tries to free the keeper block */
605 25415 : Assert(block != set->keeper);
606 : /* We shouldn't be freeing the freeblock either */
607 GBC 25415 : Assert(block != set->freeblock);
608 EUB :
609 : /* release the block from the list of blocks */
610 GIC 25415 : dlist_delete(&block->node);
611 :
612 GBC 25415 : ((MemoryContext) set)->mem_allocated -= block->blksize;
613 :
614 : #ifdef CLOBBER_FREED_MEMORY
615 GIC 25415 : wipe_mem(block, block->blksize);
616 : #endif
617 ECB :
618 GIC 25415 : free(block);
619 25415 : }
620 :
621 : /*
622 : * GenerationFree
623 : * Update number of chunks in the block, and if all chunks in the block
624 ECB : * are now free then discard the block.
625 : */
626 : void
627 GNC 1775647 : GenerationFree(void *pointer)
628 ECB : {
629 GNC 1775647 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
630 : GenerationBlock *block;
631 : GenerationContext *set;
632 : #if (defined(MEMORY_CONTEXT_CHECKING) && defined(USE_ASSERT_CHECKING)) \
633 : || defined(CLOBBER_FREED_MEMORY)
634 : Size chunksize;
635 : #endif
636 :
637 1775647 : if (MemoryChunkIsExternal(chunk))
638 : {
639 UNC 0 : block = ExternalChunkGetBlock(chunk);
640 :
641 : /*
642 : * Try to verify that we have a sane block pointer: the block header
643 : * should reference a generation context.
644 : */
645 0 : if (!GenerationBlockIsValid(block))
646 0 : elog(ERROR, "could not find block containing chunk %p", chunk);
647 :
648 : #if (defined(MEMORY_CONTEXT_CHECKING) && defined(USE_ASSERT_CHECKING)) \
649 : || defined(CLOBBER_FREED_MEMORY)
650 0 : chunksize = block->endptr - (char *) pointer;
651 : #endif
652 : }
653 : else
654 : {
655 GNC 1775647 : block = MemoryChunkGetBlock(chunk);
656 :
657 : /*
658 : * In this path, for speed reasons we just Assert that the referenced
659 : * block is good. Future field experience may show that this Assert
660 : * had better become a regular runtime test-and-elog check.
661 : */
662 1775647 : Assert(GenerationBlockIsValid(block));
663 :
664 : #if (defined(MEMORY_CONTEXT_CHECKING) && defined(USE_ASSERT_CHECKING)) \
665 : || defined(CLOBBER_FREED_MEMORY)
666 1775647 : chunksize = MemoryChunkGetValue(chunk);
667 : #endif
668 : }
669 :
670 : /* Allow access to private part of chunk header. */
671 : VALGRIND_MAKE_MEM_DEFINED(chunk, GENERATIONCHUNK_PRIVATE_LEN);
672 :
673 ECB : #ifdef MEMORY_CONTEXT_CHECKING
674 EUB : /* Test for someone scribbling on unused space in chunk */
675 GNC 1775647 : Assert(chunk->requested_size < chunksize);
676 1775647 : if (!sentinel_ok(pointer, chunk->requested_size))
677 UNC 0 : elog(WARNING, "detected write past chunk end in %s %p",
678 : ((MemoryContext) block->context)->name, chunk);
679 ECB : #endif
680 :
681 : #ifdef CLOBBER_FREED_MEMORY
682 GNC 1775647 : wipe_mem(pointer, chunksize);
683 : #endif
684 ECB :
685 : #ifdef MEMORY_CONTEXT_CHECKING
686 : /* Reset requested_size to InvalidAllocSize in freed chunks */
687 GNC 1775647 : chunk->requested_size = InvalidAllocSize;
688 : #endif
689 :
690 CBC 1775647 : block->nfree += 1;
691 ECB :
692 GIC 1775647 : Assert(block->nchunks > 0);
693 CBC 1775647 : Assert(block->nfree <= block->nchunks);
694 :
695 : /* If there are still allocated chunks in the block, we're done. */
696 1775647 : if (block->nfree < block->nchunks)
697 GIC 1770426 : return;
698 ECB :
699 GNC 5221 : set = block->context;
700 :
701 ECB : /* Don't try to free the keeper block, just mark it empty */
702 GIC 5221 : if (block == set->keeper)
703 : {
704 5221 : GenerationBlockMarkEmpty(block);
705 5221 : return;
706 : }
707 :
708 : /*
709 EUB : * If there is no freeblock set or if this is the freeblock then instead
710 : * of freeing this memory, we keep it around so that new allocations have
711 : * the option of recycling it.
712 : */
713 UBC 0 : if (set->freeblock == NULL || set->freeblock == block)
714 EUB : {
715 : /* XXX should we only recycle maxBlockSize sized blocks? */
716 UIC 0 : set->freeblock = block;
717 0 : GenerationBlockMarkEmpty(block);
718 UBC 0 : return;
719 EUB : }
720 :
721 : /* Also make sure the block is not marked as the current block. */
722 UIC 0 : if (set->block == block)
723 0 : set->block = NULL;
724 :
725 EUB : /*
726 : * The block is empty, so let's get rid of it. First remove it from the
727 : * list of blocks, then return it to malloc().
728 : */
729 UIC 0 : dlist_delete(&block->node);
730 :
731 UNC 0 : set->header.mem_allocated -= block->blksize;
732 UIC 0 : free(block);
733 : }
734 :
735 : /*
736 : * GenerationRealloc
737 : * When handling repalloc, we simply allocate a new chunk, copy the data
738 EUB : * and discard the old one. The only exception is when the new size fits
739 : * into the old chunk - in that case we just update chunk header.
740 : */
741 : void *
742 UNC 0 : GenerationRealloc(void *pointer, Size size)
743 : {
744 0 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
745 : GenerationContext *set;
746 : GenerationBlock *block;
747 : GenerationPointer newPointer;
748 : Size oldsize;
749 :
750 EUB : /* Allow access to private part of chunk header. */
751 : VALGRIND_MAKE_MEM_DEFINED(chunk, GENERATIONCHUNK_PRIVATE_LEN);
752 :
753 UNC 0 : if (MemoryChunkIsExternal(chunk))
754 : {
755 0 : block = ExternalChunkGetBlock(chunk);
756 :
757 : /*
758 : * Try to verify that we have a sane block pointer: the block header
759 : * should reference a generation context.
760 : */
761 0 : if (!GenerationBlockIsValid(block))
762 0 : elog(ERROR, "could not find block containing chunk %p", chunk);
763 :
764 0 : oldsize = block->endptr - (char *) pointer;
765 : }
766 : else
767 : {
768 0 : block = MemoryChunkGetBlock(chunk);
769 :
770 : /*
771 : * In this path, for speed reasons we just Assert that the referenced
772 : * block is good. Future field experience may show that this Assert
773 : * had better become a regular runtime test-and-elog check.
774 : */
775 0 : Assert(GenerationBlockIsValid(block));
776 :
777 0 : oldsize = MemoryChunkGetValue(chunk);
778 : }
779 :
780 0 : set = block->context;
781 :
782 : #ifdef MEMORY_CONTEXT_CHECKING
783 : /* Test for someone scribbling on unused space in chunk */
784 0 : Assert(chunk->requested_size < oldsize);
785 0 : if (!sentinel_ok(pointer, chunk->requested_size))
786 0 : elog(WARNING, "detected write past chunk end in %s %p",
787 : ((MemoryContext) set)->name, chunk);
788 EUB : #endif
789 :
790 : /*
791 : * Maybe the allocated area already is >= the new size. (In particular,
792 : * we always fall out here if the requested size is a decrease.)
793 : *
794 : * This memory context does not use power-of-2 chunk sizing and instead
795 : * carves the chunks to be as small as possible, so most repalloc() calls
796 : * will end up in the palloc/memcpy/pfree branch.
797 : *
798 : * XXX Perhaps we should annotate this condition with unlikely()?
799 : */
800 UIC 0 : if (oldsize >= size)
801 EUB : {
802 : #ifdef MEMORY_CONTEXT_CHECKING
803 UIC 0 : Size oldrequest = chunk->requested_size;
804 EUB :
805 : #ifdef RANDOMIZE_ALLOCATED_MEMORY
806 : /* We can only fill the extra space if we know the prior request */
807 : if (size > oldrequest)
808 : randomize_mem((char *) pointer + oldrequest,
809 : size - oldrequest);
810 : #endif
811 :
812 UIC 0 : chunk->requested_size = size;
813 :
814 : /*
815 : * If this is an increase, mark any newly-available part UNDEFINED.
816 : * Otherwise, mark the obsolete part NOACCESS.
817 : */
818 : if (size > oldrequest)
819 : VALGRIND_MAKE_MEM_UNDEFINED((char *) pointer + oldrequest,
820 : size - oldrequest);
821 : else
822 : VALGRIND_MAKE_MEM_NOACCESS((char *) pointer + size,
823 : oldsize - size);
824 EUB :
825 : /* set mark to catch clobber of "unused" space */
826 UNC 0 : set_sentinel(pointer, size);
827 : #else /* !MEMORY_CONTEXT_CHECKING */
828 :
829 : /*
830 : * We don't have the information to determine whether we're growing
831 : * the old request or shrinking it, so we conservatively mark the
832 : * entire new allocation DEFINED.
833 : */
834 : VALGRIND_MAKE_MEM_NOACCESS(pointer, oldsize);
835 EUB : VALGRIND_MAKE_MEM_DEFINED(pointer, size);
836 : #endif
837 :
838 : /* Disallow external access to private part of chunk header. */
839 : VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN);
840 :
841 UIC 0 : return pointer;
842 : }
843 :
844 : /* allocate new chunk */
845 0 : newPointer = GenerationAlloc((MemoryContext) set, size);
846 :
847 : /* leave immediately if request was not completed */
848 0 : if (newPointer == NULL)
849 EUB : {
850 : /* Disallow external access to private part of chunk header. */
851 : VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN);
852 UIC 0 : return NULL;
853 : }
854 :
855 : /*
856 : * GenerationAlloc() may have returned a region that is still NOACCESS.
857 : * Change it to UNDEFINED for the moment; memcpy() will then transfer
858 : * definedness from the old allocation to the new. If we know the old
859 : * allocation, copy just that much. Otherwise, make the entire old chunk
860 : * defined to avoid errors as we copy the currently-NOACCESS trailing
861 : * bytes.
862 : */
863 : VALGRIND_MAKE_MEM_UNDEFINED(newPointer, size);
864 EUB : #ifdef MEMORY_CONTEXT_CHECKING
865 UIC 0 : oldsize = chunk->requested_size;
866 : #else
867 : VALGRIND_MAKE_MEM_DEFINED(pointer, oldsize);
868 EUB : #endif
869 :
870 : /* transfer existing data (certain to fit) */
871 UBC 0 : memcpy(newPointer, pointer, oldsize);
872 :
873 : /* free old chunk */
874 UNC 0 : GenerationFree(pointer);
875 EUB :
876 UIC 0 : return newPointer;
877 : }
878 :
879 : /*
880 : * GenerationGetChunkContext
881 : * Return the MemoryContext that 'pointer' belongs to.
882 : */
883 : MemoryContext
884 UNC 0 : GenerationGetChunkContext(void *pointer)
885 : {
886 0 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
887 : GenerationBlock *block;
888 :
889 0 : if (MemoryChunkIsExternal(chunk))
890 0 : block = ExternalChunkGetBlock(chunk);
891 : else
892 0 : block = (GenerationBlock *) MemoryChunkGetBlock(chunk);
893 :
894 0 : Assert(GenerationBlockIsValid(block));
895 0 : return &block->context->header;
896 : }
897 :
898 : /*
899 : * GenerationGetChunkSpace
900 : * Given a currently-allocated chunk, determine the total space
901 : * it occupies (including all memory-allocation overhead).
902 : */
903 : Size
904 GNC 18960853 : GenerationGetChunkSpace(void *pointer)
905 : {
906 18960853 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
907 : Size chunksize;
908 :
909 18960853 : if (MemoryChunkIsExternal(chunk))
910 : {
911 84 : GenerationBlock *block = ExternalChunkGetBlock(chunk);
912 :
913 84 : Assert(GenerationBlockIsValid(block));
914 84 : chunksize = block->endptr - (char *) pointer;
915 : }
916 : else
917 18960769 : chunksize = MemoryChunkGetValue(chunk);
918 :
919 18960853 : return Generation_CHUNKHDRSZ + chunksize;
920 EUB : }
921 :
922 : /*
923 : * GenerationIsEmpty
924 : * Is a GenerationContext empty of any allocated space?
925 : */
926 : bool
927 UIC 0 : GenerationIsEmpty(MemoryContext context)
928 : {
929 0 : GenerationContext *set = (GenerationContext *) context;
930 : dlist_iter iter;
931 :
932 UNC 0 : Assert(GenerationIsValid(set));
933 :
934 UIC 0 : dlist_foreach(iter, &set->blocks)
935 EUB : {
936 UIC 0 : GenerationBlock *block = dlist_container(GenerationBlock, node, iter.cur);
937 EUB :
938 UIC 0 : if (block->nchunks > 0)
939 0 : return false;
940 EUB : }
941 :
942 UIC 0 : return true;
943 EUB : }
944 :
945 : /*
946 : * GenerationStats
947 : * Compute stats about memory consumption of a Generation context.
948 : *
949 : * printfunc: if not NULL, pass a human-readable stats string to this.
950 : * passthru: pass this pointer through to printfunc.
951 : * totals: if not NULL, add stats about this context into *totals.
952 : * print_to_stderr: print stats to stderr if true, elog otherwise.
953 : *
954 : * XXX freespace only accounts for empty space at the end of the block, not
955 ECB : * space of freed chunks (which is unknown).
956 : */
957 : void
958 UIC 0 : GenerationStats(MemoryContext context,
959 : MemoryStatsPrintFunc printfunc, void *passthru,
960 ECB : MemoryContextCounters *totals, bool print_to_stderr)
961 : {
962 LBC 0 : GenerationContext *set = (GenerationContext *) context;
963 UIC 0 : Size nblocks = 0;
964 LBC 0 : Size nchunks = 0;
965 0 : Size nfreechunks = 0;
966 : Size totalspace;
967 UIC 0 : Size freespace = 0;
968 ECB : dlist_iter iter;
969 :
970 UNC 0 : Assert(GenerationIsValid(set));
971 :
972 ECB : /* Include context header in totalspace */
973 UIC 0 : totalspace = MAXALIGN(sizeof(GenerationContext));
974 :
975 0 : dlist_foreach(iter, &set->blocks)
976 : {
977 0 : GenerationBlock *block = dlist_container(GenerationBlock, node, iter.cur);
978 :
979 0 : nblocks++;
980 UBC 0 : nchunks += block->nchunks;
981 UIC 0 : nfreechunks += block->nfree;
982 UBC 0 : totalspace += block->blksize;
983 UIC 0 : freespace += (block->endptr - block->freeptr);
984 : }
985 EUB :
986 UIC 0 : if (printfunc)
987 EUB : {
988 : char stats_string[200];
989 :
990 UIC 0 : snprintf(stats_string, sizeof(stats_string),
991 EUB : "%zu total in %zu blocks (%zu chunks); %zu free (%zu chunks); %zu used",
992 : totalspace, nblocks, nchunks, freespace,
993 : nfreechunks, totalspace - freespace);
994 UIC 0 : printfunc(context, passthru, stats_string, print_to_stderr);
995 EUB : }
996 :
997 UIC 0 : if (totals)
998 : {
999 0 : totals->nblocks += nblocks;
1000 0 : totals->freechunks += nfreechunks;
1001 0 : totals->totalspace += totalspace;
1002 0 : totals->freespace += freespace;
1003 : }
1004 0 : }
1005 :
1006 :
1007 : #ifdef MEMORY_CONTEXT_CHECKING
1008 :
1009 : /*
1010 : * GenerationCheck
1011 EUB : * Walk through chunks and check consistency of memory.
1012 : *
1013 : * NOTE: report errors as WARNING, *not* ERROR or FATAL. Otherwise you'll
1014 : * find yourself in an infinite loop when trouble occurs, because this
1015 : * routine will be entered again when elog cleanup tries to release memory!
1016 : */
1017 : void
1018 GBC 192419 : GenerationCheck(MemoryContext context)
1019 : {
1020 192419 : GenerationContext *gen = (GenerationContext *) context;
1021 GIC 192419 : const char *name = context->name;
1022 : dlist_iter iter;
1023 GBC 192419 : Size total_allocated = 0;
1024 :
1025 : /* walk all blocks in this context */
1026 410620 : dlist_foreach(iter, &gen->blocks)
1027 : {
1028 218201 : GenerationBlock *block = dlist_container(GenerationBlock, node, iter.cur);
1029 : int nfree,
1030 EUB : nchunks;
1031 : char *ptr;
1032 GNC 218201 : bool has_external_chunk = false;
1033 EUB :
1034 GBC 218201 : total_allocated += block->blksize;
1035 EUB :
1036 : /*
1037 : * nfree > nchunks is surely wrong. Equality is allowed as the block
1038 : * might completely empty if it's the freeblock.
1039 : */
1040 GBC 218201 : if (block->nfree > block->nchunks)
1041 UIC 0 : elog(WARNING, "problem in Generation %s: number of free chunks %d in block %p exceeds %d allocated",
1042 : name, block->nfree, block, block->nchunks);
1043 :
1044 : /* check block belongs to the correct context */
1045 GNC 218201 : if (block->context != gen)
1046 UNC 0 : elog(WARNING, "problem in Generation %s: bogus context link in block %p",
1047 : name, block);
1048 :
1049 EUB : /* Now walk through the chunks and count them. */
1050 GIC 218201 : nfree = 0;
1051 218201 : nchunks = 0;
1052 218201 : ptr = ((char *) block) + Generation_BLOCKHDRSZ;
1053 EUB :
1054 GIC 17608161 : while (ptr < block->freeptr)
1055 : {
1056 GNC 17389960 : MemoryChunk *chunk = (MemoryChunk *) ptr;
1057 : GenerationBlock *chunkblock;
1058 : Size chunksize;
1059 :
1060 EUB : /* Allow access to private part of chunk header. */
1061 : VALGRIND_MAKE_MEM_DEFINED(chunk, GENERATIONCHUNK_PRIVATE_LEN);
1062 :
1063 GNC 17389960 : if (MemoryChunkIsExternal(chunk))
1064 : {
1065 84 : chunkblock = ExternalChunkGetBlock(chunk);
1066 84 : chunksize = block->endptr - (char *) MemoryChunkGetPointer(chunk);
1067 84 : has_external_chunk = true;
1068 : }
1069 : else
1070 : {
1071 17389876 : chunkblock = MemoryChunkGetBlock(chunk);
1072 17389876 : chunksize = MemoryChunkGetValue(chunk);
1073 : }
1074 :
1075 EUB : /* move to the next chunk */
1076 GNC 17389960 : ptr += (chunksize + Generation_CHUNKHDRSZ);
1077 EUB :
1078 GIC 17389960 : nchunks += 1;
1079 :
1080 : /* chunks have both block and context pointers, so check both */
1081 GNC 17389960 : if (chunkblock != block)
1082 UIC 0 : elog(WARNING, "problem in Generation %s: bogus block link in block %p, chunk %p",
1083 : name, block, chunk);
1084 :
1085 :
1086 ECB : /* is chunk allocated? */
1087 GNC 17389960 : if (chunk->requested_size != InvalidAllocSize)
1088 : {
1089 : /* now make sure the chunk size is correct */
1090 17389701 : if (chunksize < chunk->requested_size ||
1091 17389701 : chunksize != MAXALIGN(chunksize))
1092 UNC 0 : elog(WARNING, "problem in Generation %s: bogus chunk size in block %p, chunk %p",
1093 : name, block, chunk);
1094 :
1095 : /* check sentinel */
1096 GNC 17389701 : Assert(chunk->requested_size < chunksize);
1097 17389701 : if (!sentinel_ok(chunk, Generation_CHUNKHDRSZ + chunk->requested_size))
1098 LBC 0 : elog(WARNING, "problem in Generation %s: detected write past chunk end in block %p, chunk %p",
1099 : name, block, chunk);
1100 : }
1101 : else
1102 GIC 259 : nfree += 1;
1103 :
1104 ECB : /*
1105 EUB : * If chunk is allocated, disallow external access to private part
1106 : * of chunk header.
1107 : */
1108 GNC 17389960 : if (chunk->requested_size != InvalidAllocSize)
1109 ECB : VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN);
1110 EUB : }
1111 :
1112 : /*
1113 : * Make sure we got the expected number of allocated and free chunks
1114 ECB : * (as tracked in the block header).
1115 : */
1116 CBC 218201 : if (nchunks != block->nchunks)
1117 UIC 0 : elog(WARNING, "problem in Generation %s: number of allocated chunks %d in block %p does not match header %d",
1118 ECB : name, nchunks, block, block->nchunks);
1119 :
1120 CBC 218201 : if (nfree != block->nfree)
1121 UIC 0 : elog(WARNING, "problem in Generation %s: number of free chunks %d in block %p does not match header %d",
1122 : name, nfree, block, block->nfree);
1123 :
1124 GNC 218201 : if (has_external_chunk && nchunks > 1)
1125 UNC 0 : elog(WARNING, "problem in Generation %s: external chunk on non-dedicated block %p",
1126 : name, block);
1127 :
1128 : }
1129 :
1130 GIC 192419 : Assert(total_allocated == context->mem_allocated);
1131 192419 : }
1132 ECB :
1133 : #endif /* MEMORY_CONTEXT_CHECKING */
|
