Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * bump.c
4 : : * Bump allocator definitions.
5 : : *
6 : : * Bump is a MemoryContext implementation designed for memory usages which
7 : : * require allocating a large number of chunks, none of which ever need to be
8 : : * pfree'd or realloc'd. Chunks allocated by this context have no chunk header
9 : : * and operations which ordinarily require looking at the chunk header cannot
10 : : * be performed. For example, pfree, realloc, GetMemoryChunkSpace and
11 : : * GetMemoryChunkContext are all not possible with bump allocated chunks. The
12 : : * only way to release memory allocated by this context type is to reset or
13 : : * delete the context.
14 : : *
15 : : * Portions Copyright (c) 2024, PostgreSQL Global Development Group
16 : : *
17 : : * IDENTIFICATION
18 : : * src/backend/utils/mmgr/bump.c
19 : : *
20 : : *
21 : : * Bump is best suited to cases which require a large number of short-lived
22 : : * chunks where performance matters. Because bump allocated chunks don't
23 : : * have a chunk header, it can fit more chunks on each block. This means we
24 : : * can do more with less memory and fewer cache lines. The reason it's best
25 : : * suited for short-lived usages of memory is that ideally, pointers to bump
26 : : * allocated chunks won't be visible to a large amount of code. The more
27 : : * code that operates on memory allocated by this allocator, the more chances
28 : : * that some code will try to perform a pfree or one of the other operations
29 : : * which are made impossible due to the lack of chunk header. In order to
30 : : * detect accidental usage of the various disallowed operations, we do add a
31 : : * MemoryChunk chunk header in MEMORY_CONTEXT_CHECKING builds and have the
32 : : * various disallowed functions raise an ERROR.
33 : : *
34 : : * Allocations are MAXALIGNed.
35 : : *
36 : : *-------------------------------------------------------------------------
37 : : */
38 : :
39 : : #include "postgres.h"
40 : :
41 : : #include "lib/ilist.h"
42 : : #include "port/pg_bitutils.h"
43 : : #include "utils/memdebug.h"
44 : : #include "utils/memutils.h"
45 : : #include "utils/memutils_memorychunk.h"
46 : : #include "utils/memutils_internal.h"
47 : :
48 : : #define Bump_BLOCKHDRSZ MAXALIGN(sizeof(BumpBlock))
49 : :
50 : : /* No chunk header unless built with MEMORY_CONTEXT_CHECKING */
51 : : #ifdef MEMORY_CONTEXT_CHECKING
52 : : #define Bump_CHUNKHDRSZ sizeof(MemoryChunk)
53 : : #else
54 : : #define Bump_CHUNKHDRSZ 0
55 : : #endif
56 : :
57 : : #define Bump_CHUNK_FRACTION 8
58 : :
59 : : /* The keeper block is allocated in the same allocation as the set */
60 : : #define KeeperBlock(set) ((BumpBlock *) ((char *) (set) + \
61 : : MAXALIGN(sizeof(BumpContext))))
62 : : #define IsKeeperBlock(set, blk) (KeeperBlock(set) == (blk))
63 : :
64 : : typedef struct BumpBlock BumpBlock; /* forward reference */
65 : :
66 : : typedef struct BumpContext
67 : : {
68 : : MemoryContextData header; /* Standard memory-context fields */
69 : :
70 : : /* Bump context parameters */
71 : : uint32 initBlockSize; /* initial block size */
72 : : uint32 maxBlockSize; /* maximum block size */
73 : : uint32 nextBlockSize; /* next block size to allocate */
74 : : uint32 allocChunkLimit; /* effective chunk size limit */
75 : :
76 : : dlist_head blocks; /* list of blocks with the block currently
77 : : * being filled at the head */
78 : : } BumpContext;
79 : :
80 : : /*
81 : : * BumpBlock
82 : : * BumpBlock is the unit of memory that is obtained by bump.c from
83 : : * malloc(). It contains zero or more allocations, which are the
84 : : * units requested by palloc().
85 : : */
86 : : struct BumpBlock
87 : : {
88 : : dlist_node node; /* doubly-linked list of blocks */
89 : : #ifdef MEMORY_CONTEXT_CHECKING
90 : : BumpContext *context; /* pointer back to the owning context */
91 : : #endif
92 : : char *freeptr; /* start of free space in this block */
93 : : char *endptr; /* end of space in this block */
94 : : };
95 : :
96 : : /*
97 : : * BumpIsValid
98 : : * True iff set is valid bump context.
99 : : */
100 : : #define BumpIsValid(set) \
101 : : (PointerIsValid(set) && IsA(set, BumpContext))
102 : :
103 : : /*
104 : : * We always store external chunks on a dedicated block. This makes fetching
105 : : * the block from an external chunk easy since it's always the first and only
106 : : * chunk on the block.
107 : : */
108 : : #define ExternalChunkGetBlock(chunk) \
109 : : (BumpBlock *) ((char *) chunk - Bump_BLOCKHDRSZ)
110 : :
111 : : /* Inlined helper functions */
112 : : static inline void BumpBlockInit(BumpContext *context, BumpBlock *block,
113 : : Size blksize);
114 : : static inline bool BumpBlockIsEmpty(BumpBlock *block);
115 : : static inline void BumpBlockMarkEmpty(BumpBlock *block);
116 : : static inline Size BumpBlockFreeBytes(BumpBlock *block);
117 : : static inline void BumpBlockFree(BumpContext *set, BumpBlock *block);
118 : :
119 : :
120 : : /*
121 : : * BumpContextCreate
122 : : * Create a new Bump context.
123 : : *
124 : : * parent: parent context, or NULL if top-level context
125 : : * name: name of context (must be statically allocated)
126 : : * minContextSize: minimum context size
127 : : * initBlockSize: initial allocation block size
128 : : * maxBlockSize: maximum allocation block size
129 : : */
130 : : MemoryContext
6 drowley@postgresql.o 131 :GNC 202059 : BumpContextCreate(MemoryContext parent, const char *name, Size minContextSize,
132 : : Size initBlockSize, Size maxBlockSize)
133 : : {
134 : : Size firstBlockSize;
135 : : Size allocSize;
136 : : BumpContext *set;
137 : : BumpBlock *block;
138 : :
139 : : /* ensure MemoryChunk's size is properly maxaligned */
140 : : StaticAssertDecl(Bump_CHUNKHDRSZ == MAXALIGN(Bump_CHUNKHDRSZ),
141 : : "sizeof(MemoryChunk) is not maxaligned");
142 : :
143 : : /*
144 : : * First, validate allocation parameters. Asserts seem sufficient because
145 : : * nobody varies their parameters at runtime. We somewhat arbitrarily
146 : : * enforce a minimum 1K block size. We restrict the maximum block size to
147 : : * MEMORYCHUNK_MAX_BLOCKOFFSET as MemoryChunks are limited to this in
148 : : * regards to addressing the offset between the chunk and the block that
149 : : * the chunk is stored on. We would be unable to store the offset between
150 : : * the chunk and block for any chunks that were beyond
151 : : * MEMORYCHUNK_MAX_BLOCKOFFSET bytes into the block if the block was to be
152 : : * larger than this.
153 : : */
154 [ + - - + ]: 202059 : Assert(initBlockSize == MAXALIGN(initBlockSize) &&
155 : : initBlockSize >= 1024);
156 [ + - + - : 202059 : Assert(maxBlockSize == MAXALIGN(maxBlockSize) &&
- + ]
157 : : maxBlockSize >= initBlockSize &&
158 : : AllocHugeSizeIsValid(maxBlockSize)); /* must be safe to double */
159 [ - + - - : 202059 : Assert(minContextSize == 0 ||
- - - - ]
160 : : (minContextSize == MAXALIGN(minContextSize) &&
161 : : minContextSize >= 1024 &&
162 : : minContextSize <= maxBlockSize));
163 [ - + ]: 202059 : Assert(maxBlockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET);
164 : :
165 : : /* Determine size of initial block */
166 : 202059 : allocSize = MAXALIGN(sizeof(BumpContext)) + Bump_BLOCKHDRSZ +
167 : : Bump_CHUNKHDRSZ;
168 [ - + ]: 202059 : if (minContextSize != 0)
6 drowley@postgresql.o 169 :UNC 0 : allocSize = Max(allocSize, minContextSize);
170 : : else
6 drowley@postgresql.o 171 :GNC 202059 : allocSize = Max(allocSize, initBlockSize);
172 : :
173 : : /*
174 : : * Allocate the initial block. Unlike other bump.c blocks, it starts with
175 : : * the context header and its block header follows that.
176 : : */
177 : 202059 : set = (BumpContext *) malloc(allocSize);
178 [ - + ]: 202059 : if (set == NULL)
179 : : {
6 drowley@postgresql.o 180 :UNC 0 : MemoryContextStats(TopMemoryContext);
181 [ # # ]: 0 : ereport(ERROR,
182 : : (errcode(ERRCODE_OUT_OF_MEMORY),
183 : : errmsg("out of memory"),
184 : : errdetail("Failed while creating memory context \"%s\".",
185 : : name)));
186 : : }
187 : :
188 : : /*
189 : : * Avoid writing code that can fail between here and MemoryContextCreate;
190 : : * we'd leak the header and initial block if we ereport in this stretch.
191 : : */
6 drowley@postgresql.o 192 :GNC 202059 : dlist_init(&set->blocks);
193 : :
194 : : /* Fill in the initial block's block header */
195 : 202059 : block = KeeperBlock(set);
196 : : /* determine the block size and initialize it */
197 : 202059 : firstBlockSize = allocSize - MAXALIGN(sizeof(BumpContext));
198 : 202059 : BumpBlockInit(set, block, firstBlockSize);
199 : :
200 : : /* add it to the doubly-linked list of blocks */
201 : 202059 : dlist_push_head(&set->blocks, &block->node);
202 : :
203 : : /*
204 : : * Fill in BumpContext-specific header fields. The Asserts above should
205 : : * ensure that these all fit inside a uint32.
206 : : */
207 : 202059 : set->initBlockSize = (uint32) initBlockSize;
208 : 202059 : set->maxBlockSize = (uint32) maxBlockSize;
209 : 202059 : set->nextBlockSize = (uint32) initBlockSize;
210 : :
211 : : /*
212 : : * Compute the allocation chunk size limit for this context.
213 : : *
214 : : * Limit the maximum size a non-dedicated chunk can be so that we can fit
215 : : * at least Bump_CHUNK_FRACTION of chunks this big onto the maximum sized
216 : : * block. We must further limit this value so that it's no more than
217 : : * MEMORYCHUNK_MAX_VALUE. We're unable to have non-external chunks larger
218 : : * than that value as we store the chunk size in the MemoryChunk 'value'
219 : : * field in the call to MemoryChunkSetHdrMask().
220 : : */
221 : 202059 : set->allocChunkLimit = Min(maxBlockSize, MEMORYCHUNK_MAX_VALUE);
222 : 202059 : while ((Size) (set->allocChunkLimit + Bump_CHUNKHDRSZ) >
223 [ + + ]: 1010295 : (Size) ((Size) (maxBlockSize - Bump_BLOCKHDRSZ) / Bump_CHUNK_FRACTION))
224 : 808236 : set->allocChunkLimit >>= 1;
225 : :
226 : : /* Finally, do the type-independent part of context creation */
227 : 202059 : MemoryContextCreate((MemoryContext) set, T_BumpContext, MCTX_BUMP_ID,
228 : : parent, name);
229 : :
230 : 202059 : ((MemoryContext) set)->mem_allocated = allocSize;
231 : :
232 : 202059 : return (MemoryContext) set;
233 : : }
234 : :
235 : : /*
236 : : * BumpReset
237 : : * Frees all memory which is allocated in the given set.
238 : : *
239 : : * The code simply frees all the blocks in the context apart from the keeper
240 : : * block.
241 : : */
242 : : void
243 : 203119 : BumpReset(MemoryContext context)
244 : : {
245 : 203119 : BumpContext *set = (BumpContext *) context;
246 : : dlist_mutable_iter miter;
247 : :
248 [ + - - + ]: 203119 : Assert(BumpIsValid(set));
249 : :
250 : : #ifdef MEMORY_CONTEXT_CHECKING
251 : : /* Check for corruption and leaks before freeing */
252 : 203119 : BumpCheck(context);
253 : : #endif
254 : :
255 [ + - + + ]: 418760 : dlist_foreach_modify(miter, &set->blocks)
256 : : {
257 : 215641 : BumpBlock *block = dlist_container(BumpBlock, node, miter.cur);
258 : :
259 [ + + ]: 215641 : if (IsKeeperBlock(set, block))
260 : 203119 : BumpBlockMarkEmpty(block);
261 : : else
262 : 12522 : BumpBlockFree(set, block);
263 : : }
264 : :
265 : : /* Reset block size allocation sequence, too */
266 : 203119 : set->nextBlockSize = set->initBlockSize;
267 : :
268 : : /* Ensure there is only 1 item in the dlist */
269 [ - + ]: 203119 : Assert(!dlist_is_empty(&set->blocks));
270 [ - + ]: 203119 : Assert(!dlist_has_next(&set->blocks, dlist_head_node(&set->blocks)));
271 : 203119 : }
272 : :
273 : : /*
274 : : * BumpDelete
275 : : * Free all memory which is allocated in the given context.
276 : : */
277 : : void
278 : 202059 : BumpDelete(MemoryContext context)
279 : : {
280 : : /* Reset to release all releasable BumpBlocks */
281 : 202059 : BumpReset(context);
282 : : /* And free the context header and keeper block */
283 : 202059 : free(context);
284 : 202059 : }
285 : :
286 : : /*
287 : : * Helper for BumpAlloc() that allocates an entire block for the chunk.
288 : : *
289 : : * BumpAlloc()'s comment explains why this is separate.
290 : : */
291 : : pg_noinline
292 : : static void *
6 drowley@postgresql.o 293 :UNC 0 : BumpAllocLarge(MemoryContext context, Size size, int flags)
294 : : {
295 : 0 : BumpContext *set = (BumpContext *) context;
296 : : BumpBlock *block;
297 : : #ifdef MEMORY_CONTEXT_CHECKING
298 : : MemoryChunk *chunk;
299 : : #endif
300 : : Size chunk_size;
301 : : Size required_size;
302 : : Size blksize;
303 : :
304 : : /* validate 'size' is within the limits for the given 'flags' */
305 : 0 : MemoryContextCheckSize(context, size, flags);
306 : :
307 : : #ifdef MEMORY_CONTEXT_CHECKING
308 : : /* ensure there's always space for the sentinel byte */
309 : 0 : chunk_size = MAXALIGN(size + 1);
310 : : #else
311 : : chunk_size = MAXALIGN(size);
312 : : #endif
313 : :
314 : 0 : required_size = chunk_size + Bump_CHUNKHDRSZ;
315 : 0 : blksize = required_size + Bump_BLOCKHDRSZ;
316 : :
317 : 0 : block = (BumpBlock *) malloc(blksize);
318 [ # # ]: 0 : if (block == NULL)
319 : 0 : return NULL;
320 : :
321 : 0 : context->mem_allocated += blksize;
322 : :
323 : : /* the block is completely full */
324 : 0 : block->freeptr = block->endptr = ((char *) block) + blksize;
325 : :
326 : : #ifdef MEMORY_CONTEXT_CHECKING
327 : : /* block with a single (used) chunk */
328 : 0 : block->context = set;
329 : :
330 : 0 : chunk = (MemoryChunk *) (((char *) block) + Bump_BLOCKHDRSZ);
331 : :
332 : : /* mark the MemoryChunk as externally managed */
333 : 0 : MemoryChunkSetHdrMaskExternal(chunk, MCTX_BUMP_ID);
334 : :
335 : 0 : chunk->requested_size = size;
336 : : /* set mark to catch clobber of "unused" space */
337 [ # # ]: 0 : Assert(size < chunk_size);
338 : 0 : set_sentinel(MemoryChunkGetPointer(chunk), size);
339 : : #endif
340 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
341 : : /* fill the allocated space with junk */
342 : : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
343 : : #endif
344 : :
345 : : /* add the block to the list of allocated blocks */
346 : 0 : dlist_push_head(&set->blocks, &block->node);
347 : :
348 : : #ifdef MEMORY_CONTEXT_CHECKING
349 : : /* Ensure any padding bytes are marked NOACCESS. */
350 : : VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
351 : : chunk_size - size);
352 : :
353 : : /* Disallow access to the chunk header. */
354 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, Bump_CHUNKHDRSZ);
355 : :
356 : 0 : return MemoryChunkGetPointer(chunk);
357 : : #else
358 : : return (void *) (((char *) block) + Bump_BLOCKHDRSZ);
359 : : #endif
360 : : }
361 : :
362 : : /*
363 : : * Small helper for allocating a new chunk from a chunk, to avoid duplicating
364 : : * the code between BumpAlloc() and BumpAllocFromNewBlock().
365 : : */
366 : : static inline void *
6 drowley@postgresql.o 367 :GNC 11352074 : BumpAllocChunkFromBlock(MemoryContext context, BumpBlock *block, Size size,
368 : : Size chunk_size)
369 : : {
370 : : #ifdef MEMORY_CONTEXT_CHECKING
371 : : MemoryChunk *chunk;
372 : : #else
373 : : void *ptr;
374 : : #endif
375 : :
376 : : /* validate we've been given a block with enough free space */
377 [ - + ]: 11352074 : Assert(block != NULL);
378 [ - + ]: 11352074 : Assert((block->endptr - block->freeptr) >= Bump_CHUNKHDRSZ + chunk_size);
379 : :
380 : : #ifdef MEMORY_CONTEXT_CHECKING
381 : 11352074 : chunk = (MemoryChunk *) block->freeptr;
382 : : #else
383 : : ptr = (void *) block->freeptr;
384 : : #endif
385 : :
386 : : /* point the freeptr beyond this chunk */
387 : 11352074 : block->freeptr += (Bump_CHUNKHDRSZ + chunk_size);
388 [ - + ]: 11352074 : Assert(block->freeptr <= block->endptr);
389 : :
390 : : #ifdef MEMORY_CONTEXT_CHECKING
391 : : /* Prepare to initialize the chunk header. */
392 : : VALGRIND_MAKE_MEM_UNDEFINED(chunk, Bump_CHUNKHDRSZ);
393 : :
394 : 11352074 : MemoryChunkSetHdrMask(chunk, block, chunk_size, MCTX_BUMP_ID);
395 : 11352074 : chunk->requested_size = size;
396 : : /* set mark to catch clobber of "unused" space */
397 [ - + ]: 11352074 : Assert(size < chunk_size);
398 : 11352074 : set_sentinel(MemoryChunkGetPointer(chunk), size);
399 : :
400 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
401 : : /* fill the allocated space with junk */
402 : : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
403 : : #endif
404 : :
405 : : /* Ensure any padding bytes are marked NOACCESS. */
406 : : VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
407 : : chunk_size - size);
408 : :
409 : : /* Disallow access to the chunk header. */
410 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, Bump_CHUNKHDRSZ);
411 : :
412 : 11352074 : return MemoryChunkGetPointer(chunk);
413 : : #else
414 : : return ptr;
415 : : #endif /* MEMORY_CONTEXT_CHECKING */
416 : : }
417 : :
418 : : /*
419 : : * Helper for BumpAlloc() that allocates a new block and returns a chunk
420 : : * allocated from it.
421 : : *
422 : : * BumpAlloc()'s comment explains why this is separate.
423 : : */
424 : : pg_noinline
425 : : static void *
426 : 12522 : BumpAllocFromNewBlock(MemoryContext context, Size size, int flags,
427 : : Size chunk_size)
428 : : {
429 : 12522 : BumpContext *set = (BumpContext *) context;
430 : : BumpBlock *block;
431 : : Size blksize;
432 : : Size required_size;
433 : :
434 : : /*
435 : : * The first such block has size initBlockSize, and we double the space in
436 : : * each succeeding block, but not more than maxBlockSize.
437 : : */
438 : 12522 : blksize = set->nextBlockSize;
439 : 12522 : set->nextBlockSize <<= 1;
440 [ + + ]: 12522 : if (set->nextBlockSize > set->maxBlockSize)
441 : 4 : set->nextBlockSize = set->maxBlockSize;
442 : :
443 : : /* we'll need space for the chunk, chunk hdr and block hdr */
444 : 12522 : required_size = chunk_size + Bump_CHUNKHDRSZ + Bump_BLOCKHDRSZ;
445 : : /* round the size up to the next power of 2 */
446 [ - + ]: 12522 : if (blksize < required_size)
6 drowley@postgresql.o 447 :UNC 0 : blksize = pg_nextpower2_size_t(required_size);
448 : :
6 drowley@postgresql.o 449 :GNC 12522 : block = (BumpBlock *) malloc(blksize);
450 : :
451 [ - + ]: 12522 : if (block == NULL)
6 drowley@postgresql.o 452 :UNC 0 : return MemoryContextAllocationFailure(context, size, flags);
453 : :
6 drowley@postgresql.o 454 :GNC 12522 : context->mem_allocated += blksize;
455 : :
456 : : /* initialize the new block */
457 : 12522 : BumpBlockInit(set, block, blksize);
458 : :
459 : : /* add it to the doubly-linked list of blocks */
460 : 12522 : dlist_push_head(&set->blocks, &block->node);
461 : :
462 : 12522 : return BumpAllocChunkFromBlock(context, block, size, chunk_size);
463 : : }
464 : :
465 : : /*
466 : : * BumpAlloc
467 : : * Returns a pointer to allocated memory of given size or raises an ERROR
468 : : * on allocation failure, or returns NULL when flags contains
469 : : * MCXT_ALLOC_NO_OOM.
470 : : *
471 : : * No request may exceed:
472 : : * MAXALIGN_DOWN(SIZE_MAX) - Bump_BLOCKHDRSZ - Bump_CHUNKHDRSZ
473 : : * All callers use a much-lower limit.
474 : : *
475 : : *
476 : : * Note: when using valgrind, it doesn't matter how the returned allocation
477 : : * is marked, as mcxt.c will set it to UNDEFINED.
478 : : * This function should only contain the most common code paths. Everything
479 : : * else should be in pg_noinline helper functions, thus avoiding the overhead
480 : : * of creating a stack frame for the common cases. Allocating memory is often
481 : : * a bottleneck in many workloads, so avoiding stack frame setup is
482 : : * worthwhile. Helper functions should always directly return the newly
483 : : * allocated memory so that we can just return that address directly as a tail
484 : : * call.
485 : : */
486 : : void *
487 : 11352074 : BumpAlloc(MemoryContext context, Size size, int flags)
488 : : {
489 : 11352074 : BumpContext *set = (BumpContext *) context;
490 : : BumpBlock *block;
491 : : Size chunk_size;
492 : : Size required_size;
493 : :
494 [ + - - + ]: 11352074 : Assert(BumpIsValid(set));
495 : :
496 : : #ifdef MEMORY_CONTEXT_CHECKING
497 : : /* ensure there's always space for the sentinel byte */
498 : 11352074 : chunk_size = MAXALIGN(size + 1);
499 : : #else
500 : : chunk_size = MAXALIGN(size);
501 : : #endif
502 : :
503 : : /*
504 : : * If requested size exceeds maximum for chunks we hand the the request
505 : : * off to BumpAllocLarge().
506 : : */
507 [ - + ]: 11352074 : if (chunk_size > set->allocChunkLimit)
6 drowley@postgresql.o 508 :UNC 0 : return BumpAllocLarge(context, size, flags);
509 : :
6 drowley@postgresql.o 510 :GNC 11352074 : required_size = chunk_size + Bump_CHUNKHDRSZ;
511 : :
512 : : /*
513 : : * Not an oversized chunk. We try to first make use of the latest block,
514 : : * but if there's not enough space in it we must allocate a new block.
515 : : */
516 : 11352074 : block = dlist_container(BumpBlock, node, dlist_head_node(&set->blocks));
517 : :
518 [ + + ]: 11352074 : if (BumpBlockFreeBytes(block) < required_size)
519 : 12522 : return BumpAllocFromNewBlock(context, size, flags, chunk_size);
520 : :
521 : : /* The current block has space, so just allocate chunk there. */
522 : 11339552 : return BumpAllocChunkFromBlock(context, block, size, chunk_size);
523 : : }
524 : :
525 : : /*
526 : : * BumpBlockInit
527 : : * Initializes 'block' assuming 'blksize'. Does not update the context's
528 : : * mem_allocated field.
529 : : */
530 : : static inline void
531 : 214581 : BumpBlockInit(BumpContext *context, BumpBlock *block, Size blksize)
532 : : {
533 : : #ifdef MEMORY_CONTEXT_CHECKING
534 : 214581 : block->context = context;
535 : : #endif
536 : 214581 : block->freeptr = ((char *) block) + Bump_BLOCKHDRSZ;
537 : 214581 : block->endptr = ((char *) block) + blksize;
538 : :
539 : : /* Mark unallocated space NOACCESS. */
540 : : VALGRIND_MAKE_MEM_NOACCESS(block->freeptr, blksize - Bump_BLOCKHDRSZ);
541 : 214581 : }
542 : :
543 : : /*
544 : : * BumpBlockIsEmpty
545 : : * Returns true iff 'block' contains no chunks
546 : : */
547 : : static inline bool
6 drowley@postgresql.o 548 :UNC 0 : BumpBlockIsEmpty(BumpBlock *block)
549 : : {
550 : : /* it's empty if the freeptr has not moved */
551 : 0 : return (block->freeptr == ((char *) block + Bump_BLOCKHDRSZ));
552 : : }
553 : :
554 : : /*
555 : : * BumpBlockMarkEmpty
556 : : * Set a block as empty. Does not free the block.
557 : : */
558 : : static inline void
6 drowley@postgresql.o 559 :GNC 203119 : BumpBlockMarkEmpty(BumpBlock *block)
560 : : {
561 : : #if defined(USE_VALGRIND) || defined(CLOBBER_FREED_MEMORY)
562 : 203119 : char *datastart = ((char *) block) + Bump_BLOCKHDRSZ;
563 : : #endif
564 : :
565 : : #ifdef CLOBBER_FREED_MEMORY
566 : 203119 : wipe_mem(datastart, block->freeptr - datastart);
567 : : #else
568 : : /* wipe_mem() would have done this */
569 : : VALGRIND_MAKE_MEM_NOACCESS(datastart, block->freeptr - datastart);
570 : : #endif
571 : :
572 : : /* Reset the block, but don't return it to malloc */
573 : 203119 : block->freeptr = ((char *) block) + Bump_BLOCKHDRSZ;
574 : 203119 : }
575 : :
576 : : /*
577 : : * BumpBlockFreeBytes
578 : : * Returns the number of bytes free in 'block'
579 : : */
580 : : static inline Size
581 : 11352074 : BumpBlockFreeBytes(BumpBlock *block)
582 : : {
583 : 11352074 : return (block->endptr - block->freeptr);
584 : : }
585 : :
586 : : /*
587 : : * BumpBlockFree
588 : : * Remove 'block' from 'set' and release the memory consumed by it.
589 : : */
590 : : static inline void
591 : 12522 : BumpBlockFree(BumpContext *set, BumpBlock *block)
592 : : {
593 : : /* Make sure nobody tries to free the keeper block */
594 [ - + ]: 12522 : Assert(!IsKeeperBlock(set, block));
595 : :
596 : : /* release the block from the list of blocks */
597 : 12522 : dlist_delete(&block->node);
598 : :
599 : 12522 : ((MemoryContext) set)->mem_allocated -= ((char *) block->endptr - (char *) block);
600 : :
601 : : #ifdef CLOBBER_FREED_MEMORY
602 : 12522 : wipe_mem(block, ((char *) block->endptr - (char *) block));
603 : : #endif
604 : :
605 : 12522 : free(block);
606 : 12522 : }
607 : :
608 : : /*
609 : : * BumpFree
610 : : * Unsupported.
611 : : */
612 : : void
6 drowley@postgresql.o 613 :UNC 0 : BumpFree(void *pointer)
614 : : {
615 [ # # ]: 0 : elog(ERROR, "pfree is not supported by the bump memory allocator");
616 : : }
617 : :
618 : : /*
619 : : * BumpRealloc
620 : : * Unsupported.
621 : : */
622 : : void *
623 : 0 : BumpRealloc(void *pointer, Size size, int flags)
624 : : {
625 [ # # ]: 0 : elog(ERROR, "%s is not supported by the bump memory allocator", "realloc");
626 : : return NULL; /* keep compiler quiet */
627 : : }
628 : :
629 : : /*
630 : : * BumpGetChunkContext
631 : : * Unsupported.
632 : : */
633 : : MemoryContext
634 : 0 : BumpGetChunkContext(void *pointer)
635 : : {
636 [ # # ]: 0 : elog(ERROR, "%s is not supported by the bump memory allocator", "GetMemoryChunkContext");
637 : : return NULL; /* keep compiler quiet */
638 : : }
639 : :
640 : : /*
641 : : * BumpGetChunkSpace
642 : : * Given a currently-allocated chunk, determine the total space
643 : : * it occupies (including all memory-allocation overhead).
644 : : */
645 : : Size
646 : 0 : BumpGetChunkSpace(void *pointer)
647 : : {
648 [ # # ]: 0 : elog(ERROR, "%s is not supported by the bump memory allocator", "GetMemoryChunkSpace");
649 : : return 0; /* keep compiler quiet */
650 : : }
651 : :
652 : : /*
653 : : * BumpIsEmpty
654 : : * Is a BumpContext empty of any allocated space?
655 : : */
656 : : bool
657 : 0 : BumpIsEmpty(MemoryContext context)
658 : : {
659 : 0 : BumpContext *set = (BumpContext *) context;
660 : : dlist_iter iter;
661 : :
662 [ # # # # ]: 0 : Assert(BumpIsValid(set));
663 : :
664 [ # # # # ]: 0 : dlist_foreach(iter, &set->blocks)
665 : : {
666 : 0 : BumpBlock *block = dlist_container(BumpBlock, node, iter.cur);
667 : :
668 [ # # ]: 0 : if (!BumpBlockIsEmpty(block))
669 : 0 : return false;
670 : : }
671 : :
672 : 0 : return true;
673 : : }
674 : :
675 : : /*
676 : : * BumpStats
677 : : * Compute stats about memory consumption of a Bump context.
678 : : *
679 : : * printfunc: if not NULL, pass a human-readable stats string to this.
680 : : * passthru: pass this pointer through to printfunc.
681 : : * totals: if not NULL, add stats about this context into *totals.
682 : : * print_to_stderr: print stats to stderr if true, elog otherwise.
683 : : */
684 : : void
685 : 0 : BumpStats(MemoryContext context, MemoryStatsPrintFunc printfunc,
686 : : void *passthru, MemoryContextCounters *totals, bool print_to_stderr)
687 : : {
688 : 0 : BumpContext *set = (BumpContext *) context;
689 : 0 : Size nblocks = 0;
690 : 0 : Size totalspace = 0;
691 : 0 : Size freespace = 0;
692 : : dlist_iter iter;
693 : :
694 [ # # # # ]: 0 : Assert(BumpIsValid(set));
695 : :
696 [ # # # # ]: 0 : dlist_foreach(iter, &set->blocks)
697 : : {
698 : 0 : BumpBlock *block = dlist_container(BumpBlock, node, iter.cur);
699 : :
700 : 0 : nblocks++;
701 : 0 : totalspace += (block->endptr - (char *) block);
702 : 0 : freespace += (block->endptr - block->freeptr);
703 : : }
704 : :
705 [ # # ]: 0 : if (printfunc)
706 : : {
707 : : char stats_string[200];
708 : :
709 : 0 : snprintf(stats_string, sizeof(stats_string),
710 : : "%zu total in %zu blocks; %zu free; %zu used",
711 : : totalspace, nblocks, freespace, totalspace - freespace);
712 : 0 : printfunc(context, passthru, stats_string, print_to_stderr);
713 : : }
714 : :
715 [ # # ]: 0 : if (totals)
716 : : {
717 : 0 : totals->nblocks += nblocks;
718 : 0 : totals->totalspace += totalspace;
719 : 0 : totals->freespace += freespace;
720 : : }
721 : 0 : }
722 : :
723 : :
724 : : #ifdef MEMORY_CONTEXT_CHECKING
725 : :
726 : : /*
727 : : * BumpCheck
728 : : * Walk through chunks and check consistency of memory.
729 : : *
730 : : * NOTE: report errors as WARNING, *not* ERROR or FATAL. Otherwise you'll
731 : : * find yourself in an infinite loop when trouble occurs, because this
732 : : * routine will be entered again when elog cleanup tries to release memory!
733 : : */
734 : : void
6 drowley@postgresql.o 735 :GNC 203448 : BumpCheck(MemoryContext context)
736 : : {
737 : 203448 : BumpContext *bump = (BumpContext *) context;
738 : 203448 : const char *name = context->name;
739 : : dlist_iter iter;
740 : 203448 : Size total_allocated = 0;
741 : :
742 : : /* walk all blocks in this context */
743 [ + - + + ]: 420082 : dlist_foreach(iter, &bump->blocks)
744 : : {
745 : 216634 : BumpBlock *block = dlist_container(BumpBlock, node, iter.cur);
746 : : int nchunks;
747 : : char *ptr;
748 : 216634 : bool has_external_chunk = false;
749 : :
750 [ + + ]: 216634 : if (IsKeeperBlock(bump, block))
751 : 203448 : total_allocated += block->endptr - (char *) bump;
752 : : else
753 : 13186 : total_allocated += block->endptr - (char *) block;
754 : :
755 : : /* check block belongs to the correct context */
756 [ - + ]: 216634 : if (block->context != bump)
6 drowley@postgresql.o 757 [ # # ]:UNC 0 : elog(WARNING, "problem in Bump %s: bogus context link in block %p",
758 : : name, block);
759 : :
760 : : /* now walk through the chunks and count them */
6 drowley@postgresql.o 761 :GNC 216634 : nchunks = 0;
762 : 216634 : ptr = ((char *) block) + Bump_BLOCKHDRSZ;
763 : :
764 [ + + ]: 12531809 : while (ptr < block->freeptr)
765 : : {
766 : 12315175 : MemoryChunk *chunk = (MemoryChunk *) ptr;
767 : : BumpBlock *chunkblock;
768 : : Size chunksize;
769 : :
770 : : /* allow access to the chunk header */
771 : : VALGRIND_MAKE_MEM_DEFINED(chunk, Bump_CHUNKHDRSZ);
772 : :
773 [ - + ]: 12315175 : if (MemoryChunkIsExternal(chunk))
774 : : {
6 drowley@postgresql.o 775 :UNC 0 : chunkblock = ExternalChunkGetBlock(chunk);
776 : 0 : chunksize = block->endptr - (char *) MemoryChunkGetPointer(chunk);
777 : 0 : has_external_chunk = true;
778 : : }
779 : : else
780 : : {
6 drowley@postgresql.o 781 :GNC 12315175 : chunkblock = MemoryChunkGetBlock(chunk);
782 : 12315175 : chunksize = MemoryChunkGetValue(chunk);
783 : : }
784 : :
785 : : /* move to the next chunk */
786 : 12315175 : ptr += (chunksize + Bump_CHUNKHDRSZ);
787 : :
788 : 12315175 : nchunks += 1;
789 : :
790 : : /* chunks have both block and context pointers, so check both */
791 [ - + ]: 12315175 : if (chunkblock != block)
6 drowley@postgresql.o 792 [ # # ]:UNC 0 : elog(WARNING, "problem in Bump %s: bogus block link in block %p, chunk %p",
793 : : name, block, chunk);
794 : : }
795 : :
6 drowley@postgresql.o 796 [ - + - - ]:GNC 216634 : if (has_external_chunk && nchunks > 1)
6 drowley@postgresql.o 797 [ # # ]:UNC 0 : elog(WARNING, "problem in Bump %s: external chunk on non-dedicated block %p",
798 : : name, block);
799 : :
800 : : }
801 : :
6 drowley@postgresql.o 802 [ - + ]:GNC 203448 : Assert(total_allocated == context->mem_allocated);
803 : 203448 : }
804 : :
805 : : #endif /* MEMORY_CONTEXT_CHECKING */
|
