Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit_deform.c
4 : * Generate code for deforming a heap tuple.
5 : *
6 : * This gains performance benefits over unJITed deforming from compile-time
7 : * knowledge of the tuple descriptor. Fixed column widths, NOT NULLness, etc
8 : * can be taken advantage of.
9 : *
10 : * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
11 : * Portions Copyright (c) 1994, Regents of the University of California
12 : *
13 : * IDENTIFICATION
14 : * src/backend/jit/llvm/llvmjit_deform.c
15 : *
16 : *-------------------------------------------------------------------------
17 : */
18 :
19 : #include "postgres.h"
20 :
21 : #include <llvm-c/Core.h>
22 :
23 : #include "access/htup_details.h"
24 : #include "access/tupdesc_details.h"
25 : #include "executor/tuptable.h"
26 : #include "jit/llvmjit.h"
27 : #include "jit/llvmjit_emit.h"
28 :
29 :
30 : /*
31 : * Create a function that deforms a tuple of type desc up to natts columns.
32 : */
33 : LLVMValueRef
1606 andres 34 CBC 3134 : slot_compile_deform(LLVMJitContext *context, TupleDesc desc,
35 : const TupleTableSlotOps *ops, int natts)
36 : {
37 : char *funcname;
38 :
39 : LLVMModuleRef mod;
40 : LLVMBuilderRef b;
41 :
42 : LLVMTypeRef deform_sig;
43 : LLVMValueRef v_deform_fn;
44 :
45 : LLVMBasicBlockRef b_entry;
46 : LLVMBasicBlockRef b_adjust_unavail_cols;
47 : LLVMBasicBlockRef b_find_start;
48 :
49 : LLVMBasicBlockRef b_out;
50 : LLVMBasicBlockRef b_dead;
51 : LLVMBasicBlockRef *attcheckattnoblocks;
52 : LLVMBasicBlockRef *attstartblocks;
53 : LLVMBasicBlockRef *attisnullblocks;
54 : LLVMBasicBlockRef *attcheckalignblocks;
55 : LLVMBasicBlockRef *attalignblocks;
56 : LLVMBasicBlockRef *attstoreblocks;
57 :
58 : LLVMValueRef v_offp;
59 :
60 : LLVMValueRef v_tupdata_base;
61 : LLVMValueRef v_tts_values;
62 : LLVMValueRef v_tts_nulls;
63 : LLVMValueRef v_slotoffp;
64 : LLVMValueRef v_flagsp;
65 : LLVMValueRef v_nvalidp;
66 : LLVMValueRef v_nvalid;
67 : LLVMValueRef v_maxatt;
68 :
69 : LLVMValueRef v_slot;
70 :
71 : LLVMValueRef v_tupleheaderp;
72 : LLVMValueRef v_tuplep;
73 : LLVMValueRef v_infomask1;
74 : LLVMValueRef v_infomask2;
75 : LLVMValueRef v_bits;
76 :
77 : LLVMValueRef v_hoff;
78 :
79 : LLVMValueRef v_hasnulls;
80 :
81 : /* last column (0 indexed) guaranteed to exist */
1840 82 3134 : int guaranteed_column_number = -1;
83 :
84 : /* current known alignment */
85 3134 : int known_alignment = 0;
86 :
87 : /* if true, known_alignment describes definite offset of column */
88 3134 : bool attguaranteedalign = true;
89 :
90 : int attnum;
91 :
92 : /* virtual tuples never need deforming, so don't generate code */
1606 93 3134 : if (ops == &TTSOpsVirtual)
1606 andres 94 UBC 0 : return NULL;
95 :
96 : /* decline to JIT for slot types we don't know to handle */
1605 andres 97 CBC 3134 : if (ops != &TTSOpsHeapTuple && ops != &TTSOpsBufferHeapTuple &&
98 : ops != &TTSOpsMinimalTuple)
1605 andres 99 UBC 0 : return NULL;
100 :
1840 andres 101 CBC 3134 : mod = llvm_mutable_module(context);
102 :
103 3134 : funcname = llvm_expand_funcname(context, "deform");
104 :
105 : /*
106 : * Check which columns have to exist, so we don't have to check the row's
107 : * natts unnecessarily.
108 : */
109 17903 : for (attnum = 0; attnum < desc->natts; attnum++)
110 : {
1839 111 14769 : Form_pg_attribute att = TupleDescAttr(desc, attnum);
112 :
113 : /*
114 : * If the column is declared NOT NULL then it must be present in every
115 : * tuple, unless there's a "missing" entry that could provide a
116 : * non-NULL value for it. That in turn guarantees that the NULL bitmap
117 : * - if there are any NULLable columns - is at least long enough to
118 : * cover columns up to attnum.
119 : *
120 : * Be paranoid and also check !attisdropped, even though the
121 : * combination of attisdropped && attnotnull combination shouldn't
122 : * exist.
123 : */
1440 124 14769 : if (att->attnotnull &&
125 3574 : !att->atthasmissing &&
126 3574 : !att->attisdropped)
1840 127 3574 : guaranteed_column_number = attnum;
128 : }
129 :
130 : /* Create the signature and function */
131 : {
132 : LLVMTypeRef param_types[1];
133 :
134 3134 : param_types[0] = l_ptr(StructTupleTableSlot);
135 :
136 3134 : deform_sig = LLVMFunctionType(LLVMVoidType(), param_types,
137 : lengthof(param_types), 0);
138 : }
139 3134 : v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig);
140 3134 : LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage);
141 3134 : LLVMSetParamAlignment(LLVMGetParam(v_deform_fn, 0), MAXIMUM_ALIGNOF);
142 3134 : llvm_copy_attributes(AttributeTemplate, v_deform_fn);
143 :
144 : b_entry =
145 3134 : LLVMAppendBasicBlock(v_deform_fn, "entry");
146 : b_adjust_unavail_cols =
147 3134 : LLVMAppendBasicBlock(v_deform_fn, "adjust_unavail_cols");
148 : b_find_start =
149 3134 : LLVMAppendBasicBlock(v_deform_fn, "find_startblock");
150 : b_out =
151 3134 : LLVMAppendBasicBlock(v_deform_fn, "outblock");
152 : b_dead =
153 3134 : LLVMAppendBasicBlock(v_deform_fn, "deadblock");
154 :
155 3134 : b = LLVMCreateBuilder();
156 :
157 3134 : attcheckattnoblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
158 3134 : attstartblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
159 3134 : attisnullblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
160 3134 : attcheckalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
161 3134 : attalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
162 3134 : attstoreblocks = palloc(sizeof(LLVMBasicBlockRef) * natts);
163 :
164 3134 : known_alignment = 0;
165 :
166 3134 : LLVMPositionBuilderAtEnd(b, b_entry);
167 :
168 : /* perform allocas first, llvm only converts those to registers */
169 3134 : v_offp = LLVMBuildAlloca(b, TypeSizeT, "v_offp");
170 :
171 3134 : v_slot = LLVMGetParam(v_deform_fn, 0);
172 :
173 : v_tts_values =
174 3134 : l_load_struct_gep(b, v_slot, FIELDNO_TUPLETABLESLOT_VALUES,
175 : "tts_values");
176 : v_tts_nulls =
177 3134 : l_load_struct_gep(b, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL,
178 : "tts_ISNULL");
1637 179 3134 : v_flagsp = LLVMBuildStructGEP(b, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, "");
1840 180 3134 : v_nvalidp = LLVMBuildStructGEP(b, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, "");
181 :
1605 182 3134 : if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple)
183 1907 : {
184 : LLVMValueRef v_heapslot;
185 :
186 : v_heapslot =
187 1907 : LLVMBuildBitCast(b,
188 : v_slot,
189 : l_ptr(StructHeapTupleTableSlot),
190 : "heapslot");
191 1907 : v_slotoffp = LLVMBuildStructGEP(b, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, "");
192 : v_tupleheaderp =
193 1907 : l_load_struct_gep(b, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE,
194 : "tupleheader");
195 : }
196 1227 : else if (ops == &TTSOpsMinimalTuple)
197 : {
198 : LLVMValueRef v_minimalslot;
199 :
200 : v_minimalslot =
201 1227 : LLVMBuildBitCast(b,
202 : v_slot,
203 : l_ptr(StructMinimalTupleTableSlot),
204 : "minimalslot");
205 1227 : v_slotoffp = LLVMBuildStructGEP(b, v_minimalslot, FIELDNO_MINIMALTUPLETABLESLOT_OFF, "");
206 : v_tupleheaderp =
207 1227 : l_load_struct_gep(b, v_minimalslot, FIELDNO_MINIMALTUPLETABLESLOT_TUPLE,
208 : "tupleheader");
209 : }
210 : else
211 : {
212 : /* should've returned at the start of the function */
1605 andres 213 UBC 0 : pg_unreachable();
214 : }
215 :
216 : v_tuplep =
1840 andres 217 CBC 3134 : l_load_struct_gep(b, v_tupleheaderp, FIELDNO_HEAPTUPLEDATA_DATA,
218 : "tuple");
219 : v_bits =
220 3134 : LLVMBuildBitCast(b,
221 : LLVMBuildStructGEP(b, v_tuplep,
222 : FIELDNO_HEAPTUPLEHEADERDATA_BITS,
223 : ""),
224 : l_ptr(LLVMInt8Type()),
225 : "t_bits");
226 : v_infomask1 =
227 3134 : l_load_struct_gep(b, v_tuplep,
228 : FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK,
229 : "infomask1");
230 : v_infomask2 =
231 3134 : l_load_struct_gep(b,
232 : v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2,
233 : "infomask2");
234 :
235 : /* t_infomask & HEAP_HASNULL */
236 : v_hasnulls =
237 3134 : LLVMBuildICmp(b, LLVMIntNE,
238 : LLVMBuildAnd(b,
239 : l_int16_const(HEAP_HASNULL),
240 : v_infomask1, ""),
241 : l_int16_const(0),
242 : "hasnulls");
243 :
244 : /* t_infomask2 & HEAP_NATTS_MASK */
245 3134 : v_maxatt = LLVMBuildAnd(b,
246 : l_int16_const(HEAP_NATTS_MASK),
247 : v_infomask2,
248 : "maxatt");
249 :
250 : /*
251 : * Need to zext, as getelementptr otherwise treats hoff as a signed 8bit
252 : * integer, which'd yield a negative offset for t_hoff > 127.
253 : */
254 3134 : v_hoff =
1594 255 3134 : LLVMBuildZExt(b,
256 : l_load_struct_gep(b, v_tuplep,
257 : FIELDNO_HEAPTUPLEHEADERDATA_HOFF,
258 : ""),
259 : LLVMInt32Type(), "t_hoff");
260 :
261 : v_tupdata_base =
1840 262 3134 : LLVMBuildGEP(b,
263 : LLVMBuildBitCast(b,
264 : v_tuplep,
265 : l_ptr(LLVMInt8Type()),
266 : ""),
267 : &v_hoff, 1,
268 : "v_tupdata_base");
269 :
270 : /*
271 : * Load tuple start offset from slot. Will be reset below in case there's
272 : * no existing deformed columns in slot.
273 : */
274 : {
275 : LLVMValueRef v_off_start;
276 :
277 3134 : v_off_start = LLVMBuildLoad(b, v_slotoffp, "v_slot_off");
278 3134 : v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, "");
279 3134 : LLVMBuildStore(b, v_off_start, v_offp);
280 : }
281 :
282 : /* build the basic block for each attribute, need them as jump target */
283 10072 : for (attnum = 0; attnum < natts; attnum++)
284 : {
285 13876 : attcheckattnoblocks[attnum] =
286 6938 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckattno", attnum);
287 13876 : attstartblocks[attnum] =
288 6938 : l_bb_append_v(v_deform_fn, "block.attr.%d.start", attnum);
289 13876 : attisnullblocks[attnum] =
290 6938 : l_bb_append_v(v_deform_fn, "block.attr.%d.attisnull", attnum);
291 13876 : attcheckalignblocks[attnum] =
292 6938 : l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckalign", attnum);
293 13876 : attalignblocks[attnum] =
294 6938 : l_bb_append_v(v_deform_fn, "block.attr.%d.align", attnum);
295 6938 : attstoreblocks[attnum] =
296 6938 : l_bb_append_v(v_deform_fn, "block.attr.%d.store", attnum);
297 : }
298 :
299 : /*
300 : * Check if it is guaranteed that all the desired attributes are available
301 : * in the tuple (but still possibly NULL), by dint of either the last
302 : * to-be-deformed column being NOT NULL, or subsequent ones not accessed
303 : * here being NOT NULL. If that's not guaranteed the tuple headers natt's
304 : * has to be checked, and missing attributes potentially have to be
305 : * fetched (using slot_getmissingattrs().
306 : */
307 3134 : if ((natts - 1) <= guaranteed_column_number)
308 : {
309 : /* just skip through unnecessary blocks */
310 215 : LLVMBuildBr(b, b_adjust_unavail_cols);
311 215 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
312 215 : LLVMBuildBr(b, b_find_start);
313 : }
314 : else
315 : {
316 : LLVMValueRef v_params[3];
317 :
318 : /* branch if not all columns available */
319 2919 : LLVMBuildCondBr(b,
320 : LLVMBuildICmp(b, LLVMIntULT,
321 : v_maxatt,
322 : l_int16_const(natts),
323 : ""),
324 : b_adjust_unavail_cols,
325 : b_find_start);
326 :
327 : /* if not, memset tts_isnull of relevant cols to true */
328 2919 : LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
329 :
1839 330 2919 : v_params[0] = v_slot;
331 2919 : v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32Type(), "");
332 2919 : v_params[2] = l_int32_const(natts);
1158 333 2919 : LLVMBuildCall(b, llvm_pg_func(mod, "slot_getmissingattrs"),
334 : v_params, lengthof(v_params), "");
1840 335 2919 : LLVMBuildBr(b, b_find_start);
336 : }
337 :
338 3134 : LLVMPositionBuilderAtEnd(b, b_find_start);
339 :
340 3134 : v_nvalid = LLVMBuildLoad(b, v_nvalidp, "");
341 :
342 : /*
343 : * Build switch to go from nvalid to the right startblock. Callers
344 : * currently don't have the knowledge, but it'd be good for performance to
345 : * avoid this check when it's known that the slot is empty (e.g. in scan
346 : * nodes).
347 : */
348 : if (true)
349 : {
350 3134 : LLVMValueRef v_switch = LLVMBuildSwitch(b, v_nvalid,
351 : b_dead, natts);
352 :
353 10072 : for (attnum = 0; attnum < natts; attnum++)
354 : {
1657 355 6938 : LLVMValueRef v_attno = l_int16_const(attnum);
356 :
1840 357 6938 : LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]);
358 : }
359 : }
360 : else
361 : {
362 : /* jump from entry block to first block */
363 : LLVMBuildBr(b, attcheckattnoblocks[0]);
364 : }
365 :
366 3134 : LLVMPositionBuilderAtEnd(b, b_dead);
367 3134 : LLVMBuildUnreachable(b);
368 :
369 : /*
370 : * Iterate over each attribute that needs to be deformed, build code to
371 : * deform it.
372 : */
373 10072 : for (attnum = 0; attnum < natts; attnum++)
374 : {
375 6938 : Form_pg_attribute att = TupleDescAttr(desc, attnum);
376 : LLVMValueRef v_incby;
377 : int alignto;
378 6938 : LLVMValueRef l_attno = l_int16_const(attnum);
379 : LLVMValueRef v_attdatap;
380 : LLVMValueRef v_resultp;
381 :
382 : /* build block checking whether we did all the necessary attributes */
383 6938 : LLVMPositionBuilderAtEnd(b, attcheckattnoblocks[attnum]);
384 :
385 : /*
386 : * If this is the first attribute, slot->tts_nvalid was 0. Therefore
387 : * also reset offset to 0, it may be from a previous execution.
388 : */
389 6938 : if (attnum == 0)
390 : {
391 3134 : LLVMBuildStore(b, l_sizet_const(0), v_offp);
392 : }
393 :
394 : /*
395 : * Build check whether column is available (i.e. whether the tuple has
396 : * that many columns stored). We can avoid the branch if we know
397 : * there's a subsequent NOT NULL column.
398 : */
399 6938 : if (attnum <= guaranteed_column_number)
400 : {
401 893 : LLVMBuildBr(b, attstartblocks[attnum]);
402 : }
403 : else
404 : {
405 : LLVMValueRef v_islast;
406 :
1839 407 6045 : v_islast = LLVMBuildICmp(b, LLVMIntUGE,
408 : l_attno,
409 : v_maxatt,
410 : "heap_natts");
1840 411 6045 : LLVMBuildCondBr(b, v_islast, b_out, attstartblocks[attnum]);
412 : }
413 6938 : LLVMPositionBuilderAtEnd(b, attstartblocks[attnum]);
414 :
415 : /*
416 : * Check for nulls if necessary. No need to take missing attributes
417 : * into account, because if they're present the heaptuple's natts
418 : * would have indicated that a slot_getmissingattrs() is needed.
419 : */
420 6938 : if (!att->attnotnull)
421 : {
422 : LLVMBasicBlockRef b_ifnotnull;
423 : LLVMBasicBlockRef b_ifnull;
424 : LLVMBasicBlockRef b_next;
425 : LLVMValueRef v_attisnull;
426 : LLVMValueRef v_nullbyteno;
427 : LLVMValueRef v_nullbytemask;
428 : LLVMValueRef v_nullbyte;
429 : LLVMValueRef v_nullbit;
430 :
431 6069 : b_ifnotnull = attcheckalignblocks[attnum];
432 6069 : b_ifnull = attisnullblocks[attnum];
433 :
434 6069 : if (attnum + 1 == natts)
435 2919 : b_next = b_out;
436 : else
437 3150 : b_next = attcheckattnoblocks[attnum + 1];
438 :
439 6069 : v_nullbyteno = l_int32_const(attnum >> 3);
440 6069 : v_nullbytemask = l_int8_const(1 << ((attnum) & 0x07));
441 6069 : v_nullbyte = l_load_gep1(b, v_bits, v_nullbyteno, "attnullbyte");
442 :
443 6069 : v_nullbit = LLVMBuildICmp(b,
444 : LLVMIntEQ,
445 : LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""),
446 : l_int8_const(0),
447 : "attisnull");
448 :
449 6069 : v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, "");
450 :
451 6069 : LLVMBuildCondBr(b, v_attisnull, b_ifnull, b_ifnotnull);
452 :
453 6069 : LLVMPositionBuilderAtEnd(b, b_ifnull);
454 :
455 : /* store null-byte */
456 6069 : LLVMBuildStore(b,
457 : l_int8_const(1),
458 : LLVMBuildGEP(b, v_tts_nulls, &l_attno, 1, ""));
459 : /* store zero datum */
460 6069 : LLVMBuildStore(b,
461 : l_sizet_const(0),
462 : LLVMBuildGEP(b, v_tts_values, &l_attno, 1, ""));
463 :
464 6069 : LLVMBuildBr(b, b_next);
465 6069 : attguaranteedalign = false;
466 : }
467 : else
468 : {
469 : /* nothing to do */
470 869 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
471 869 : LLVMPositionBuilderAtEnd(b, attisnullblocks[attnum]);
472 869 : LLVMBuildBr(b, attcheckalignblocks[attnum]);
473 : }
474 6938 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
475 :
476 : /* determine required alignment */
1131 tgl 477 6938 : if (att->attalign == TYPALIGN_INT)
1840 andres 478 5297 : alignto = ALIGNOF_INT;
1131 tgl 479 1641 : else if (att->attalign == TYPALIGN_CHAR)
1840 andres 480 615 : alignto = 1;
1131 tgl 481 1026 : else if (att->attalign == TYPALIGN_DOUBLE)
1840 andres 482 813 : alignto = ALIGNOF_DOUBLE;
1131 tgl 483 213 : else if (att->attalign == TYPALIGN_SHORT)
1840 andres 484 213 : alignto = ALIGNOF_SHORT;
485 : else
486 : {
1840 andres 487 UBC 0 : elog(ERROR, "unknown alignment");
488 : alignto = 0;
489 : }
490 :
491 : /* ------
492 : * Even if alignment is required, we can skip doing it if provably
493 : * unnecessary:
494 : * - first column is guaranteed to be aligned
495 : * - columns following a NOT NULL fixed width datum have known
496 : * alignment, can skip alignment computation if that known alignment
497 : * is compatible with current column.
498 : * ------
499 : */
1840 andres 500 CBC 6938 : if (alignto > 1 &&
501 3329 : (known_alignment < 0 || known_alignment != TYPEALIGN(alignto, known_alignment)))
502 : {
503 : /*
504 : * When accessing a varlena field, we have to "peek" to see if we
505 : * are looking at a pad byte or the first byte of a 1-byte-header
506 : * datum. A zero byte must be either a pad byte, or the first
507 : * byte of a correctly aligned 4-byte length word; in either case,
508 : * we can align safely. A non-zero byte must be either a 1-byte
509 : * length word, or the first byte of a correctly aligned 4-byte
510 : * length word; in either case, we need not align.
511 : */
512 3003 : if (att->attlen == -1)
513 : {
514 : LLVMValueRef v_possible_padbyte;
515 : LLVMValueRef v_ispad;
516 : LLVMValueRef v_off;
517 :
518 : /* don't know if short varlena or not */
519 611 : attguaranteedalign = false;
520 :
521 611 : v_off = LLVMBuildLoad(b, v_offp, "");
522 :
523 : v_possible_padbyte =
524 611 : l_load_gep1(b, v_tupdata_base, v_off, "padbyte");
525 : v_ispad =
526 611 : LLVMBuildICmp(b, LLVMIntEQ,
527 : v_possible_padbyte, l_int8_const(0),
528 : "ispadbyte");
529 611 : LLVMBuildCondBr(b, v_ispad,
530 611 : attalignblocks[attnum],
531 611 : attstoreblocks[attnum]);
532 : }
533 : else
534 : {
535 2392 : LLVMBuildBr(b, attalignblocks[attnum]);
536 : }
537 :
538 3003 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
539 :
540 : /* translation of alignment code (cf TYPEALIGN()) */
541 : {
542 : LLVMValueRef v_off_aligned;
543 3003 : LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, "");
544 :
545 : /* ((ALIGNVAL) - 1) */
546 3003 : LLVMValueRef v_alignval = l_sizet_const(alignto - 1);
547 :
548 : /* ((uintptr_t) (LEN) + ((ALIGNVAL) - 1)) */
549 3003 : LLVMValueRef v_lh = LLVMBuildAdd(b, v_off, v_alignval, "");
550 :
551 : /* ~((uintptr_t) ((ALIGNVAL) - 1)) */
552 3003 : LLVMValueRef v_rh = l_sizet_const(~(alignto - 1));
553 :
554 3003 : v_off_aligned = LLVMBuildAnd(b, v_lh, v_rh, "aligned_offset");
555 :
556 3003 : LLVMBuildStore(b, v_off_aligned, v_offp);
557 : }
558 :
559 : /*
560 : * As alignment either was unnecessary or has been performed, we
561 : * now know the current alignment. This is only safe because this
562 : * value isn't used for varlena and nullable columns.
563 : */
564 3003 : if (known_alignment >= 0)
565 : {
566 9 : Assert(known_alignment != 0);
567 9 : known_alignment = TYPEALIGN(alignto, known_alignment);
568 : }
569 :
570 3003 : LLVMBuildBr(b, attstoreblocks[attnum]);
571 3003 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
572 : }
573 : else
574 : {
575 3935 : LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
576 3935 : LLVMBuildBr(b, attalignblocks[attnum]);
577 3935 : LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
578 3935 : LLVMBuildBr(b, attstoreblocks[attnum]);
579 : }
580 6938 : LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
581 :
582 : /*
583 : * Store the current offset if known to be constant. That allows LLVM
584 : * to generate better code. Without that LLVM can't figure out that
585 : * the offset might be constant due to the jumps for previously
586 : * decoded columns.
587 : */
588 6938 : if (attguaranteedalign)
589 : {
590 845 : Assert(known_alignment >= 0);
591 845 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
592 : }
593 :
594 : /* compute what following columns are aligned to */
595 6938 : if (att->attlen < 0)
596 : {
597 : /* can't guarantee any alignment after variable length field */
598 1024 : known_alignment = -1;
599 1024 : attguaranteedalign = false;
600 : }
601 5914 : else if (att->attnotnull && attguaranteedalign && known_alignment >= 0)
602 : {
603 : /*
604 : * If the offset to the column was previously known, a NOT NULL &
605 : * fixed-width column guarantees that alignment is just the
606 : * previous alignment plus column width.
607 : */
608 836 : Assert(att->attlen > 0);
609 836 : known_alignment += att->attlen;
610 : }
611 5078 : else if (att->attnotnull && (att->attlen % alignto) == 0)
612 : {
613 : /*
614 : * After a NOT NULL fixed-width column with a length that is a
615 : * multiple of its alignment requirement, we know the following
616 : * column is aligned to at least the current column's alignment.
617 : */
1840 andres 618 UBC 0 : Assert(att->attlen > 0);
619 0 : known_alignment = alignto;
620 0 : Assert(known_alignment > 0);
621 0 : attguaranteedalign = false;
622 : }
623 : else
624 : {
1840 andres 625 CBC 5078 : known_alignment = -1;
626 5078 : attguaranteedalign = false;
627 : }
628 :
629 :
630 : /* compute address to load data from */
631 : {
632 6938 : LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, "");
633 :
634 6938 : v_attdatap =
635 6938 : LLVMBuildGEP(b, v_tupdata_base, &v_off, 1, "");
636 : }
637 :
638 : /* compute address to store value at */
639 6938 : v_resultp = LLVMBuildGEP(b, v_tts_values, &l_attno, 1, "");
640 :
641 : /* store null-byte (false) */
642 6938 : LLVMBuildStore(b, l_int8_const(0),
643 : LLVMBuildGEP(b, v_tts_nulls, &l_attno, 1, ""));
644 :
645 : /*
646 : * Store datum. For byval: datums copy the value, extend to Datum's
647 : * width, and store. For byref types: store pointer to data.
648 : */
649 6938 : if (att->attbyval)
650 : {
651 : LLVMValueRef v_tmp_loaddata;
652 : LLVMTypeRef vartypep =
653 5414 : LLVMPointerType(LLVMIntType(att->attlen * 8), 0);
654 :
655 : v_tmp_loaddata =
656 5414 : LLVMBuildPointerCast(b, v_attdatap, vartypep, "");
657 5414 : v_tmp_loaddata = LLVMBuildLoad(b, v_tmp_loaddata, "attr_byval");
658 5414 : v_tmp_loaddata = LLVMBuildZExt(b, v_tmp_loaddata, TypeSizeT, "");
659 :
660 5414 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
661 : }
662 : else
663 : {
664 : LLVMValueRef v_tmp_loaddata;
665 :
666 : /* store pointer */
667 : v_tmp_loaddata =
668 1524 : LLVMBuildPtrToInt(b,
669 : v_attdatap,
670 : TypeSizeT,
671 : "attr_ptr");
672 1524 : LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
673 : }
674 :
675 : /* increment data pointer */
676 6938 : if (att->attlen > 0)
677 : {
678 5914 : v_incby = l_sizet_const(att->attlen);
679 : }
680 1024 : else if (att->attlen == -1)
681 : {
682 1024 : v_incby = LLVMBuildCall(b,
683 : llvm_pg_func(mod, "varsize_any"),
684 : &v_attdatap, 1,
685 : "varsize_any");
686 1024 : l_callsite_ro(v_incby);
687 1024 : l_callsite_alwaysinline(v_incby);
688 : }
1840 andres 689 UBC 0 : else if (att->attlen == -2)
690 : {
691 0 : v_incby = LLVMBuildCall(b,
692 : llvm_pg_func(mod, "strlen"),
693 : &v_attdatap, 1, "strlen");
694 :
695 0 : l_callsite_ro(v_incby);
696 :
697 : /* add 1 for NUL byte */
698 0 : v_incby = LLVMBuildAdd(b, v_incby, l_sizet_const(1), "");
699 : }
700 : else
701 : {
702 0 : Assert(false);
703 : v_incby = NULL; /* silence compiler */
704 : }
705 :
1840 andres 706 CBC 6938 : if (attguaranteedalign)
707 : {
708 836 : Assert(known_alignment >= 0);
709 836 : LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
710 : }
711 : else
712 : {
713 6102 : LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, "");
714 :
715 6102 : v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset");
716 6102 : LLVMBuildStore(b, v_off, v_offp);
717 : }
718 :
719 : /*
720 : * jump to next block, unless last possible column, or all desired
721 : * (available) attributes have been fetched.
722 : */
723 6938 : if (attnum + 1 == natts)
724 : {
725 : /* jump out */
726 3134 : LLVMBuildBr(b, b_out);
727 : }
728 : else
729 : {
730 3804 : LLVMBuildBr(b, attcheckattnoblocks[attnum + 1]);
731 : }
732 : }
733 :
734 :
735 : /* build block that returns */
736 3134 : LLVMPositionBuilderAtEnd(b, b_out);
737 :
738 : {
739 3134 : LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, "");
740 : LLVMValueRef v_flags;
741 :
1657 742 3134 : LLVMBuildStore(b, l_int16_const(natts), v_nvalidp);
1840 743 3134 : v_off = LLVMBuildTrunc(b, v_off, LLVMInt32Type(), "");
744 3134 : LLVMBuildStore(b, v_off, v_slotoffp);
1637 745 3134 : v_flags = LLVMBuildLoad(b, v_flagsp, "tts_flags");
746 3134 : v_flags = LLVMBuildOr(b, v_flags, l_int16_const(TTS_FLAG_SLOW), "");
747 3134 : LLVMBuildStore(b, v_flags, v_flagsp);
1840 748 3134 : LLVMBuildRetVoid(b);
749 : }
750 :
751 3134 : LLVMDisposeBuilder(b);
752 :
753 3134 : return v_deform_fn;
754 : }
|
