Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit.c
4 : * Core part of the LLVM JIT provider.
5 : *
6 : * Copyright (c) 2016-2023, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/jit/llvm/llvmjit.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 :
14 : #include "postgres.h"
15 :
16 : #include <llvm-c/Analysis.h>
17 : #include <llvm-c/BitReader.h>
18 : #include <llvm-c/BitWriter.h>
19 : #include <llvm-c/Core.h>
20 : #include <llvm-c/ExecutionEngine.h>
21 : #if LLVM_VERSION_MAJOR > 11
22 : #include <llvm-c/Orc.h>
23 : #include <llvm-c/OrcEE.h>
24 : #include <llvm-c/LLJIT.h>
25 : #else
26 : #include <llvm-c/OrcBindings.h>
27 : #endif
28 : #include <llvm-c/Support.h>
29 : #include <llvm-c/Target.h>
30 : #include <llvm-c/Transforms/IPO.h>
31 : #include <llvm-c/Transforms/PassManagerBuilder.h>
32 : #include <llvm-c/Transforms/Scalar.h>
33 : #if LLVM_VERSION_MAJOR > 6
34 : #include <llvm-c/Transforms/Utils.h>
35 : #endif
36 :
37 : #include "jit/llvmjit.h"
38 : #include "jit/llvmjit_emit.h"
39 : #include "miscadmin.h"
40 : #include "portability/instr_time.h"
41 : #include "storage/ipc.h"
42 : #include "utils/memutils.h"
43 : #include "utils/resowner_private.h"
44 :
45 : /* Handle of a module emitted via ORC JIT */
46 : typedef struct LLVMJitHandle
47 : {
48 : #if LLVM_VERSION_MAJOR > 11
49 : LLVMOrcLLJITRef lljit;
50 : LLVMOrcResourceTrackerRef resource_tracker;
51 : #else
52 : LLVMOrcJITStackRef stack;
53 : LLVMOrcModuleHandle orc_handle;
54 : #endif
55 : } LLVMJitHandle;
56 :
57 :
58 : /* types & functions commonly needed for JITing */
59 : LLVMTypeRef TypeSizeT;
60 : LLVMTypeRef TypeParamBool;
61 : LLVMTypeRef TypeStorageBool;
62 : LLVMTypeRef TypePGFunction;
63 : LLVMTypeRef StructNullableDatum;
64 : LLVMTypeRef StructHeapTupleFieldsField3;
65 : LLVMTypeRef StructHeapTupleFields;
66 : LLVMTypeRef StructHeapTupleHeaderData;
67 : LLVMTypeRef StructHeapTupleDataChoice;
68 : LLVMTypeRef StructHeapTupleData;
69 : LLVMTypeRef StructMinimalTupleData;
70 : LLVMTypeRef StructItemPointerData;
71 : LLVMTypeRef StructBlockId;
72 : LLVMTypeRef StructFormPgAttribute;
73 : LLVMTypeRef StructTupleConstr;
74 : LLVMTypeRef StructTupleDescData;
75 : LLVMTypeRef StructTupleTableSlot;
76 : LLVMTypeRef StructHeapTupleTableSlot;
77 : LLVMTypeRef StructMinimalTupleTableSlot;
78 : LLVMTypeRef StructMemoryContextData;
79 : LLVMTypeRef StructPGFinfoRecord;
80 : LLVMTypeRef StructFmgrInfo;
81 : LLVMTypeRef StructFunctionCallInfoData;
82 : LLVMTypeRef StructExprContext;
83 : LLVMTypeRef StructExprEvalStep;
84 : LLVMTypeRef StructExprState;
85 : LLVMTypeRef StructAggState;
86 : LLVMTypeRef StructAggStatePerGroupData;
87 : LLVMTypeRef StructAggStatePerTransData;
88 :
89 : LLVMValueRef AttributeTemplate;
90 :
91 : LLVMModuleRef llvm_types_module = NULL;
92 :
93 : static bool llvm_session_initialized = false;
94 : static size_t llvm_generation = 0;
95 : static const char *llvm_triple = NULL;
96 : static const char *llvm_layout = NULL;
97 :
98 :
99 : static LLVMTargetRef llvm_targetref;
100 : #if LLVM_VERSION_MAJOR > 11
101 : static LLVMOrcThreadSafeContextRef llvm_ts_context;
102 : static LLVMOrcLLJITRef llvm_opt0_orc;
103 : static LLVMOrcLLJITRef llvm_opt3_orc;
104 : #else /* LLVM_VERSION_MAJOR > 11 */
105 : static LLVMOrcJITStackRef llvm_opt0_orc;
106 : static LLVMOrcJITStackRef llvm_opt3_orc;
107 : #endif /* LLVM_VERSION_MAJOR > 11 */
108 :
109 :
110 : static void llvm_release_context(JitContext *context);
111 : static void llvm_session_initialize(void);
112 : static void llvm_shutdown(int code, Datum arg);
113 : static void llvm_compile_module(LLVMJitContext *context);
114 : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
115 :
116 : static void llvm_create_types(void);
117 : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
118 :
119 : #if LLVM_VERSION_MAJOR > 11
120 : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
121 : static char *llvm_error_message(LLVMErrorRef error);
122 : #endif /* LLVM_VERSION_MAJOR > 11 */
123 :
1845 andres 124 CBC 987 : PG_MODULE_MAGIC;
125 :
126 :
127 : /*
128 : * Initialize LLVM JIT provider.
129 : */
130 : void
131 393 : _PG_jit_provider_init(JitProviderCallbacks *cb)
132 : {
133 393 : cb->reset_after_error = llvm_reset_after_error;
134 393 : cb->release_context = llvm_release_context;
1846 135 393 : cb->compile_expr = llvm_compile_expr;
1845 136 393 : }
137 :
138 : /*
139 : * Create a context for JITing work.
140 : *
141 : * The context, including subsidiary resources, will be cleaned up either when
142 : * the context is explicitly released, or when the lifetime of
143 : * CurrentResourceOwner ends (usually the end of the current [sub]xact).
144 : */
145 : LLVMJitContext *
146 839 : llvm_create_context(int jitFlags)
147 : {
148 : LLVMJitContext *context;
149 :
150 839 : llvm_assert_in_fatal_section();
151 :
152 839 : llvm_session_initialize();
153 :
154 839 : ResourceOwnerEnlargeJIT(CurrentResourceOwner);
155 :
156 839 : context = MemoryContextAllocZero(TopMemoryContext,
157 : sizeof(LLVMJitContext));
158 839 : context->base.flags = jitFlags;
159 :
160 : /* ensure cleanup */
161 839 : context->base.resowner = CurrentResourceOwner;
162 839 : ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
163 :
164 839 : return context;
165 : }
166 :
167 : /*
168 : * Release resources required by one llvm context.
169 : */
170 : static void
171 839 : llvm_release_context(JitContext *context)
172 : {
1844 173 839 : LLVMJitContext *llvm_context = (LLVMJitContext *) context;
174 : ListCell *lc;
175 :
176 : /*
177 : * When this backend is exiting, don't clean up LLVM. As an error might
178 : * have occurred from within LLVM, we do not want to risk reentering. All
179 : * resource cleanup is going to happen through process exit.
180 : */
881 181 839 : if (proc_exit_inprogress)
881 andres 182 UBC 0 : return;
183 :
573 andres 184 CBC 839 : llvm_enter_fatal_on_oom();
185 :
881 186 839 : if (llvm_context->module)
187 : {
188 235 : LLVMDisposeModule(llvm_context->module);
189 235 : llvm_context->module = NULL;
190 : }
191 :
524 tgl 192 1512 : foreach(lc, llvm_context->handles)
193 : {
194 673 : LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
195 :
196 : #if LLVM_VERSION_MAJOR > 11
197 : {
198 : LLVMOrcExecutionSessionRef ee;
199 : LLVMOrcSymbolStringPoolRef sp;
200 :
881 andres 201 673 : LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
202 673 : LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
203 :
204 : /*
205 : * Without triggering cleanup of the string pool, we'd leak
206 : * memory. It'd be sufficient to do this far less often, but in
207 : * experiments the required time was small enough to just always
208 : * do it.
209 : */
210 673 : ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
211 673 : sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
212 673 : LLVMOrcSymbolStringPoolClearDeadEntries(sp);
213 : }
214 : #else /* LLVM_VERSION_MAJOR > 11 */
215 : {
216 : LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
217 : }
218 : #endif /* LLVM_VERSION_MAJOR > 11 */
219 :
220 673 : pfree(jit_handle);
221 : }
524 tgl 222 839 : list_free(llvm_context->handles);
223 839 : llvm_context->handles = NIL;
224 : }
225 :
226 : /*
227 : * Return module which may be modified, e.g. by creating new functions.
228 : */
229 : LLVMModuleRef
1844 andres 230 8310 : llvm_mutable_module(LLVMJitContext *context)
231 : {
232 8310 : llvm_assert_in_fatal_section();
233 :
234 : /*
235 : * If there's no in-progress module, create a new one.
236 : */
237 8310 : if (!context->module)
238 : {
239 908 : context->compiled = false;
240 908 : context->module_generation = llvm_generation++;
241 908 : context->module = LLVMModuleCreateWithName("pg");
242 908 : LLVMSetTarget(context->module, llvm_triple);
243 908 : LLVMSetDataLayout(context->module, llvm_layout);
244 : }
245 :
246 8310 : return context->module;
247 : }
248 :
249 : /*
250 : * Expand function name to be non-conflicting. This should be used by code
251 : * generating code, when adding new externally visible function definitions to
252 : * a Module.
253 : */
254 : char *
255 8310 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
256 : {
257 8310 : Assert(context->module != NULL);
258 :
1657 259 8310 : context->base.instr.created_functions++;
260 :
261 : /*
262 : * Previously we used dots to separate, but turns out some tools, e.g.
263 : * GDB, don't like that and truncate name.
264 : */
1844 265 16620 : return psprintf("%s_%zu_%d",
266 : basename,
267 : context->module_generation,
268 8310 : context->counter++);
269 : }
270 :
271 : /*
272 : * Return pointer to function funcname, which has to exist. If there's pending
273 : * code to be optimized and emitted, do so first.
274 : */
275 : void *
276 3135 : llvm_get_function(LLVMJitContext *context, const char *funcname)
277 : {
278 : #if LLVM_VERSION_MAJOR > 11 || \
279 : defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
280 : ListCell *lc;
281 : #endif
282 :
283 3135 : llvm_assert_in_fatal_section();
284 :
285 : /*
286 : * If there is a pending / not emitted module, compile and emit now.
287 : * Otherwise we might not find the [correct] function.
288 : */
289 3135 : if (!context->compiled)
290 : {
291 673 : llvm_compile_module(context);
292 : }
293 :
294 : /*
295 : * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
296 : * to mangle here.
297 : */
298 :
299 : #if LLVM_VERSION_MAJOR > 11
881 300 3135 : foreach(lc, context->handles)
301 : {
302 3135 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
303 : instr_time starttime;
304 : instr_time endtime;
305 : LLVMErrorRef error;
306 : LLVMOrcJITTargetAddress addr;
307 :
308 3135 : INSTR_TIME_SET_CURRENT(starttime);
309 :
310 3135 : addr = 0;
311 3135 : error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
312 3135 : if (error)
881 andres 313 UBC 0 : elog(ERROR, "failed to look up symbol \"%s\": %s",
314 : funcname, llvm_error_message(error));
315 :
316 : /*
317 : * LLJIT only actually emits code the first time a symbol is
318 : * referenced. Thus add lookup time to emission time. That's counting
319 : * a bit more than with older LLVM versions, but unlikely to ever
320 : * matter.
321 : */
881 andres 322 CBC 3135 : INSTR_TIME_SET_CURRENT(endtime);
323 3135 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
324 : endtime, starttime);
325 :
326 3135 : if (addr)
327 3135 : return (void *) (uintptr_t) addr;
328 : }
329 : #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
330 : foreach(lc, context->handles)
331 : {
332 : LLVMOrcTargetAddress addr;
333 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
334 :
335 : addr = 0;
336 : if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
337 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
338 : if (addr)
339 : return (void *) (uintptr_t) addr;
340 : }
341 : #elif LLVM_VERSION_MAJOR < 5
342 : {
343 : LLVMOrcTargetAddress addr;
344 :
345 : if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
346 : return (void *) (uintptr_t) addr;
347 : if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
348 : return (void *) (uintptr_t) addr;
349 : }
350 : #else
351 : {
352 : LLVMOrcTargetAddress addr;
353 :
354 : if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
355 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
356 : if (addr)
357 : return (void *) (uintptr_t) addr;
358 : if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
359 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
360 : if (addr)
361 : return (void *) (uintptr_t) addr;
362 : }
363 : #endif
364 :
1844 andres 365 UBC 0 : elog(ERROR, "failed to JIT: %s", funcname);
366 :
367 : return NULL;
368 : }
369 :
370 : /*
371 : * Return type of a variable in llvmjit_types.c. This is useful to keep types
372 : * in sync between plain C and JIT related code.
373 : */
374 : LLVMTypeRef
853 andres 375 CBC 11916 : llvm_pg_var_type(const char *varname)
376 : {
377 : LLVMValueRef v_srcvar;
378 : LLVMTypeRef typ;
379 :
380 : /* this'll return a *pointer* to the global */
381 11916 : v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
382 11916 : if (!v_srcvar)
853 andres 383 UBC 0 : elog(ERROR, "variable %s not in llvmjit_types.c", varname);
384 :
385 : /* look at the contained type */
853 andres 386 CBC 11916 : typ = LLVMTypeOf(v_srcvar);
387 11916 : Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
388 11916 : typ = LLVMGetElementType(typ);
389 11916 : Assert(typ != NULL);
390 :
391 11916 : return typ;
392 : }
393 :
394 : /*
395 : * Return function type of a variable in llvmjit_types.c. This is useful to
396 : * keep function types in sync between C and JITed code.
397 : */
398 : LLVMTypeRef
399 5235 : llvm_pg_var_func_type(const char *varname)
400 : {
401 5235 : LLVMTypeRef typ = llvm_pg_var_type(varname);
402 :
403 : /* look at the contained type */
404 5235 : Assert(LLVMGetTypeKind(typ) == LLVMPointerTypeKind);
405 5235 : typ = LLVMGetElementType(typ);
406 5235 : Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMFunctionTypeKind);
407 :
408 5235 : return typ;
409 : }
410 :
411 : /*
412 : * Return declaration for a function referenced in llvmjit_types.c, adding it
413 : * to the module if necessary.
414 : *
415 : * This is used to make functions discovered via llvm_create_types() known to
416 : * the module that's currently being worked on.
417 : */
418 : LLVMValueRef
1158 419 6754 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
420 : {
421 : LLVMValueRef v_srcfn;
422 : LLVMValueRef v_fn;
423 :
424 : /* don't repeatedly add function */
425 6754 : v_fn = LLVMGetNamedFunction(mod, funcname);
1844 426 6754 : if (v_fn)
427 4873 : return v_fn;
428 :
1158 429 1881 : v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
430 :
431 1881 : if (!v_srcfn)
1158 andres 432 UBC 0 : elog(ERROR, "function %s not in llvmjit_types.c", funcname);
433 :
1844 andres 434 CBC 1881 : v_fn = LLVMAddFunction(mod,
435 : funcname,
436 : LLVMGetElementType(LLVMTypeOf(v_srcfn)));
1158 437 1881 : llvm_copy_attributes(v_srcfn, v_fn);
438 :
1844 439 1881 : return v_fn;
440 : }
441 :
442 : /*
443 : * Copy attributes from one function to another, for a specific index (an
444 : * index can reference return value, function and parameter attributes).
445 : */
446 : static void
906 447 33330 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
448 : {
449 : int num_attributes;
450 : LLVMAttributeRef *attrs;
451 :
452 33330 : num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
453 :
454 : /*
455 : * Not just for efficiency: LLVM <= 3.9 crashes when
456 : * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
457 : */
458 33330 : if (num_attributes == 0)
459 10190 : return;
460 :
1844 461 23140 : attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
906 462 23140 : LLVMGetAttributesAtIndex(v_from, index, attrs);
463 :
464 188675 : for (int attno = 0; attno < num_attributes; attno++)
465 165535 : LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
466 :
467 23140 : pfree(attrs);
468 : }
469 :
470 : /*
471 : * Copy all attributes from one function to another. I.e. function, return and
472 : * parameters will be copied.
473 : */
474 : void
475 10191 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
476 : {
477 : uint32 param_count;
478 :
479 : /* copy function attributes */
480 10191 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
481 :
482 : /* and the return value attributes */
483 10191 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
484 :
485 : /* and each function parameter's attribute */
486 10191 : param_count = LLVMCountParams(v_from);
487 :
488 23139 : for (int paramidx = 1; paramidx <= param_count; paramidx++)
489 12948 : llvm_copy_attributes_at_index(v_from, v_to, paramidx);
1844 490 10191 : }
491 :
492 : /*
493 : * Return a callable LLVMValueRef for fcinfo.
494 : */
495 : LLVMValueRef
1846 496 2800 : llvm_function_reference(LLVMJitContext *context,
497 : LLVMBuilderRef builder,
498 : LLVMModuleRef mod,
499 : FunctionCallInfo fcinfo)
500 : {
501 : char *modname;
502 : char *basename;
503 : char *funcname;
504 :
505 : LLVMValueRef v_fn;
506 :
507 2800 : fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
508 :
509 2800 : if (modname != NULL && basename != NULL)
510 : {
511 : /* external function in loadable library */
512 18 : funcname = psprintf("pgextern.%s.%s", modname, basename);
513 : }
514 2782 : else if (basename != NULL)
515 : {
516 : /* internal function */
215 drowley 517 GNC 2770 : funcname = pstrdup(basename);
518 : }
519 : else
520 : {
521 : /*
522 : * Function we don't know to handle, return pointer. We do so by
523 : * creating a global constant containing a pointer to the function.
524 : * Makes IR more readable.
525 : */
526 : LLVMValueRef v_fn_addr;
527 :
1846 andres 528 CBC 12 : funcname = psprintf("pgoidextern.%u",
529 12 : fcinfo->flinfo->fn_oid);
530 12 : v_fn = LLVMGetNamedGlobal(mod, funcname);
531 12 : if (v_fn != 0)
1846 andres 532 UBC 0 : return LLVMBuildLoad(builder, v_fn, "");
533 :
1846 andres 534 CBC 12 : v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
535 :
536 12 : v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
537 12 : LLVMSetInitializer(v_fn, v_fn_addr);
538 12 : LLVMSetGlobalConstant(v_fn, true);
881 539 12 : LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
540 12 : LLVMSetUnnamedAddr(v_fn, true);
541 :
1846 542 12 : return LLVMBuildLoad(builder, v_fn, "");
543 : }
544 :
545 : /* check if function already has been added */
546 2788 : v_fn = LLVMGetNamedFunction(mod, funcname);
547 2788 : if (v_fn != 0)
548 1192 : return v_fn;
549 :
550 1596 : v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
551 :
552 1596 : return v_fn;
553 : }
554 :
555 : /*
556 : * Optimize code in module using the flags set in context.
557 : */
558 : static void
1844 559 673 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
560 : {
561 : LLVMPassManagerBuilderRef llvm_pmb;
562 : LLVMPassManagerRef llvm_mpm;
563 : LLVMPassManagerRef llvm_fpm;
564 : LLVMValueRef func;
565 : int compile_optlevel;
566 :
567 673 : if (context->base.flags & PGJIT_OPT3)
568 500 : compile_optlevel = 3;
569 : else
570 173 : compile_optlevel = 0;
571 :
572 : /*
573 : * Have to create a new pass manager builder every pass through, as the
574 : * inliner has some per-builder state. Otherwise one ends up only inlining
575 : * a function the first time though.
576 : */
577 673 : llvm_pmb = LLVMPassManagerBuilderCreate();
578 673 : LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
579 673 : llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
580 :
581 673 : if (context->base.flags & PGJIT_OPT3)
582 : {
583 : /* TODO: Unscientifically determined threshold */
584 500 : LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
585 : }
586 : else
587 : {
588 : /* we rely on mem2reg heavily, so emit even in the O0 case */
589 173 : LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
590 : }
591 :
592 673 : LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
593 :
594 : /*
595 : * Do function level optimization. This could be moved to the point where
596 : * functions are emitted, to reduce memory usage a bit.
597 : */
598 673 : LLVMInitializeFunctionPassManager(llvm_fpm);
599 673 : for (func = LLVMGetFirstFunction(context->module);
600 17455 : func != NULL;
601 16782 : func = LLVMGetNextFunction(func))
602 16782 : LLVMRunFunctionPassManager(llvm_fpm, func);
603 673 : LLVMFinalizeFunctionPassManager(llvm_fpm);
604 673 : LLVMDisposePassManager(llvm_fpm);
605 :
606 : /*
607 : * Perform module level optimization. We do so even in the non-optimized
608 : * case, so always-inline functions etc get inlined. It's cheap enough.
609 : */
610 673 : llvm_mpm = LLVMCreatePassManager();
611 673 : LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
612 : llvm_mpm);
613 : /* always use always-inliner pass */
614 673 : if (!(context->base.flags & PGJIT_OPT3))
615 173 : LLVMAddAlwaysInlinerPass(llvm_mpm);
616 : /* if doing inlining, but no expensive optimization, add inlining pass */
1838 617 673 : if (context->base.flags & PGJIT_INLINE
618 500 : && !(context->base.flags & PGJIT_OPT3))
1838 andres 619 UBC 0 : LLVMAddFunctionInliningPass(llvm_mpm);
1844 andres 620 CBC 673 : LLVMRunPassManager(llvm_mpm, context->module);
621 673 : LLVMDisposePassManager(llvm_mpm);
622 :
623 673 : LLVMPassManagerBuilderDispose(llvm_pmb);
624 673 : }
625 :
626 : /*
627 : * Emit code for the currently pending module.
628 : */
629 : static void
630 673 : llvm_compile_module(LLVMJitContext *context)
631 : {
632 : LLVMJitHandle *handle;
633 : MemoryContext oldcontext;
634 : instr_time starttime;
635 : instr_time endtime;
636 : #if LLVM_VERSION_MAJOR > 11
637 : LLVMOrcLLJITRef compile_orc;
638 : #else
639 : LLVMOrcJITStackRef compile_orc;
640 : #endif
641 :
642 673 : if (context->base.flags & PGJIT_OPT3)
643 500 : compile_orc = llvm_opt3_orc;
644 : else
645 173 : compile_orc = llvm_opt0_orc;
646 :
647 : /* perform inlining */
1838 648 673 : if (context->base.flags & PGJIT_INLINE)
649 : {
650 500 : INSTR_TIME_SET_CURRENT(starttime);
651 500 : llvm_inline(context->module);
652 500 : INSTR_TIME_SET_CURRENT(endtime);
1657 653 500 : INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
654 : endtime, starttime);
655 : }
656 :
1844 657 673 : if (jit_dump_bitcode)
658 : {
659 : char *filename;
660 :
30 peter 661 UNC 0 : filename = psprintf("%d.%zu.bc",
662 : MyProcPid,
663 : context->module_generation);
1844 andres 664 UBC 0 : LLVMWriteBitcodeToFile(context->module, filename);
665 0 : pfree(filename);
666 : }
667 :
668 :
669 : /* optimize according to the chosen optimization settings */
1844 andres 670 CBC 673 : INSTR_TIME_SET_CURRENT(starttime);
671 673 : llvm_optimize_module(context, context->module);
672 673 : INSTR_TIME_SET_CURRENT(endtime);
1657 673 673 : INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
674 : endtime, starttime);
675 :
1844 676 673 : if (jit_dump_bitcode)
677 : {
678 : char *filename;
679 :
30 peter 680 UNC 0 : filename = psprintf("%d.%zu.optimized.bc",
681 : MyProcPid,
682 : context->module_generation);
1844 andres 683 UBC 0 : LLVMWriteBitcodeToFile(context->module, filename);
684 0 : pfree(filename);
685 : }
686 :
687 : handle = (LLVMJitHandle *)
881 andres 688 CBC 673 : MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
689 :
690 : /*
691 : * Emit the code. Note that this can, depending on the optimization
692 : * settings, take noticeable resources as code emission executes low-level
693 : * instruction combining/selection passes etc. Without optimization a
694 : * faster instruction selection mechanism is used.
695 : */
1844 696 673 : INSTR_TIME_SET_CURRENT(starttime);
697 : #if LLVM_VERSION_MAJOR > 11
698 : {
699 : LLVMOrcThreadSafeModuleRef ts_module;
700 : LLVMErrorRef error;
881 701 673 : LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
702 :
703 673 : ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
704 :
705 673 : handle->lljit = compile_orc;
706 673 : handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
707 :
708 : /*
709 : * NB: This doesn't actually emit code. That happens lazily the first
710 : * time a symbol defined in the module is requested. Due to that
711 : * llvm_get_function() also accounts for emission time.
712 : */
713 :
714 673 : context->module = NULL; /* will be owned by LLJIT */
715 673 : error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
716 : handle->resource_tracker,
717 : ts_module);
718 :
719 673 : if (error)
881 andres 720 UBC 0 : elog(ERROR, "failed to JIT module: %s",
721 : llvm_error_message(error));
722 :
881 andres 723 CBC 673 : handle->lljit = compile_orc;
724 :
725 : /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
726 : }
727 : #elif LLVM_VERSION_MAJOR > 6
728 : {
729 : handle->stack = compile_orc;
730 : if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
731 : llvm_resolve_symbol, NULL))
732 : elog(ERROR, "failed to JIT module");
733 :
734 : /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
735 : }
736 : #elif LLVM_VERSION_MAJOR > 4
737 : {
738 : LLVMSharedModuleRef smod;
739 :
740 : smod = LLVMOrcMakeSharedModule(context->module);
741 : handle->stack = compile_orc;
742 : if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
743 : llvm_resolve_symbol, NULL))
744 : elog(ERROR, "failed to JIT module");
745 :
746 : LLVMOrcDisposeSharedModuleRef(smod);
747 : }
748 : #else /* LLVM 4.0 and 3.9 */
749 : {
750 : handle->stack = compile_orc;
751 : handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
752 : llvm_resolve_symbol, NULL);
753 :
754 : LLVMDisposeModule(context->module);
755 : }
756 : #endif
757 :
1844 758 673 : INSTR_TIME_SET_CURRENT(endtime);
1657 759 673 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
760 : endtime, starttime);
761 :
1844 762 673 : context->module = NULL;
763 673 : context->compiled = true;
764 :
765 : /* remember emitted code for cleanup and lookups */
766 673 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
881 767 673 : context->handles = lappend(context->handles, handle);
1844 768 673 : MemoryContextSwitchTo(oldcontext);
769 :
770 673 : ereport(DEBUG1,
771 : (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
772 : INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
773 : INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
774 : INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
775 : errhidestmt(true),
776 : errhidecontext(true)));
1845 777 673 : }
778 :
779 : /*
780 : * Per session initialization.
781 : */
782 : static void
783 839 : llvm_session_initialize(void)
784 : {
785 : MemoryContext oldcontext;
1844 786 839 : char *error = NULL;
787 839 : char *cpu = NULL;
788 839 : char *features = NULL;
789 : LLVMTargetMachineRef opt0_tm;
790 : LLVMTargetMachineRef opt3_tm;
791 :
1845 792 839 : if (llvm_session_initialized)
793 446 : return;
794 :
795 393 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
796 :
797 393 : LLVMInitializeNativeTarget();
798 393 : LLVMInitializeNativeAsmPrinter();
799 393 : LLVMInitializeNativeAsmParser();
800 :
801 : /*
802 : * When targeting an LLVM version with opaque pointers enabled by
803 : * default, turn them off for the context we build our code in. We don't
804 : * need to do so for other contexts (e.g. llvm_ts_context). Once the IR is
805 : * generated, it carries the necessary information.
806 : */
807 : #if LLVM_VERSION_MAJOR > 14
808 : LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
809 : #endif
810 :
811 : /*
812 : * Synchronize types early, as that also includes inferring the target
813 : * triple.
814 : */
1844 815 393 : llvm_create_types();
816 :
817 393 : if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
818 : {
96 michael 819 UBC 0 : elog(FATAL, "failed to query triple %s", error);
820 : }
821 :
822 : /*
823 : * We want the generated code to use all available features. Therefore
824 : * grab the host CPU string and detect features of the current CPU. The
825 : * latter is needed because some CPU architectures default to enabling
826 : * features not all CPUs have (weird, huh).
827 : */
1844 andres 828 CBC 393 : cpu = LLVMGetHostCPUName();
829 393 : features = LLVMGetHostCPUFeatures();
830 393 : elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
831 : cpu, features);
832 :
833 : opt0_tm =
834 393 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
835 : LLVMCodeGenLevelNone,
836 : LLVMRelocDefault,
837 : LLVMCodeModelJITDefault);
838 : opt3_tm =
839 393 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
840 : LLVMCodeGenLevelAggressive,
841 : LLVMRelocDefault,
842 : LLVMCodeModelJITDefault);
843 :
844 393 : LLVMDisposeMessage(cpu);
845 393 : cpu = NULL;
846 393 : LLVMDisposeMessage(features);
847 393 : features = NULL;
848 :
849 : /* force symbols in main binary to be loaded */
850 393 : LLVMLoadLibraryPermanently(NULL);
851 :
852 : #if LLVM_VERSION_MAJOR > 11
853 : {
881 854 393 : llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
855 :
856 393 : llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
857 393 : opt0_tm = 0;
858 :
859 393 : llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
860 393 : opt3_tm = 0;
861 : }
862 : #else /* LLVM_VERSION_MAJOR > 11 */
863 : {
864 : llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
865 : llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
866 :
867 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
868 : if (jit_debugging_support)
869 : {
870 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
871 :
872 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
873 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
874 : }
875 : #endif
876 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
877 : if (jit_profiling_support)
878 : {
879 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
880 :
881 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
882 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
883 : }
884 : #endif
885 : }
886 : #endif /* LLVM_VERSION_MAJOR > 11 */
887 :
976 rhaas 888 393 : on_proc_exit(llvm_shutdown, 0);
889 :
1845 andres 890 393 : llvm_session_initialized = true;
891 :
892 393 : MemoryContextSwitchTo(oldcontext);
893 : }
894 :
895 : static void
896 393 : llvm_shutdown(int code, Datum arg)
897 : {
898 : /*
899 : * If llvm_shutdown() is reached while in a fatal-on-oom section an error
900 : * has occurred in the middle of LLVM code. It is not safe to call back
901 : * into LLVM (which is why a FATAL error was thrown).
902 : *
903 : * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
904 : * profiling data won't be written out.
905 : */
573 906 393 : if (llvm_in_fatal_on_oom())
907 : {
908 343 : Assert(proc_exit_inprogress);
909 343 : return;
910 : }
911 :
912 : #if LLVM_VERSION_MAJOR > 11
913 : {
881 914 50 : if (llvm_opt3_orc)
915 : {
916 50 : LLVMOrcDisposeLLJIT(llvm_opt3_orc);
917 50 : llvm_opt3_orc = NULL;
918 : }
919 50 : if (llvm_opt0_orc)
920 : {
921 50 : LLVMOrcDisposeLLJIT(llvm_opt0_orc);
922 50 : llvm_opt0_orc = NULL;
923 : }
924 50 : if (llvm_ts_context)
925 : {
926 50 : LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
927 50 : llvm_ts_context = NULL;
928 : }
929 : }
930 : #else /* LLVM_VERSION_MAJOR > 11 */
931 : {
932 : /* unregister profiling support, needs to be flushed to be useful */
933 :
934 : if (llvm_opt3_orc)
935 : {
936 : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
937 : if (jit_profiling_support)
938 : LLVMOrcUnregisterPerf(llvm_opt3_orc);
939 : #endif
940 : LLVMOrcDisposeInstance(llvm_opt3_orc);
941 : llvm_opt3_orc = NULL;
942 : }
943 :
944 : if (llvm_opt0_orc)
945 : {
946 : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
947 : if (jit_profiling_support)
948 : LLVMOrcUnregisterPerf(llvm_opt0_orc);
949 : #endif
950 : LLVMOrcDisposeInstance(llvm_opt0_orc);
951 : llvm_opt0_orc = NULL;
952 : }
953 : }
954 : #endif /* LLVM_VERSION_MAJOR > 11 */
955 : }
956 :
957 : /* helper for llvm_create_types, returning a function's return type */
958 : static LLVMTypeRef
1844 959 393 : load_return_type(LLVMModuleRef mod, const char *name)
960 : {
961 : LLVMValueRef value;
962 : LLVMTypeRef typ;
963 :
964 : /* this'll return a *pointer* to the function */
965 393 : value = LLVMGetNamedFunction(mod, name);
966 393 : if (!value)
1844 andres 967 UBC 0 : elog(ERROR, "function %s is unknown", name);
968 :
969 : /* get type of function pointer */
1844 andres 970 CBC 393 : typ = LLVMTypeOf(value);
971 393 : Assert(typ != NULL);
972 : /* dereference pointer */
973 393 : typ = LLVMGetElementType(typ);
974 393 : Assert(typ != NULL);
975 : /* and look at return type */
976 393 : typ = LLVMGetReturnType(typ);
977 393 : Assert(typ != NULL);
978 :
979 393 : return typ;
980 : }
981 :
982 : /*
983 : * Load required information, types, function signatures from llvmjit_types.c
984 : * and make them available in global variables.
985 : *
986 : * Those global variables are then used while emitting code.
987 : */
988 : static void
989 393 : llvm_create_types(void)
990 : {
991 : char path[MAXPGPATH];
992 : LLVMMemoryBufferRef buf;
993 : char *msg;
994 :
995 393 : snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
996 :
997 : /* open file */
998 393 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
999 : {
1844 andres 1000 UBC 0 : elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1001 : path, msg);
1002 : }
1003 :
1004 : /* eagerly load contents, going to need it all */
1158 andres 1005 CBC 393 : if (LLVMParseBitcode2(buf, &llvm_types_module))
1006 : {
1844 andres 1007 UBC 0 : elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
1008 : }
1844 andres 1009 CBC 393 : LLVMDisposeMemoryBuffer(buf);
1010 :
1011 : /*
1012 : * Load triple & layout from clang emitted file so we're guaranteed to be
1013 : * compatible.
1014 : */
1158 1015 393 : llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1016 393 : llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1017 :
853 1018 393 : TypeSizeT = llvm_pg_var_type("TypeSizeT");
1158 1019 393 : TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
853 1020 393 : TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1021 393 : TypePGFunction = llvm_pg_var_type("TypePGFunction");
1022 393 : StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1023 393 : StructExprContext = llvm_pg_var_type("StructExprContext");
1024 393 : StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1025 393 : StructExprState = llvm_pg_var_type("StructExprState");
1026 393 : StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1027 393 : StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1028 393 : StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1029 393 : StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1030 393 : StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1031 393 : StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1032 393 : StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1033 393 : StructAggState = llvm_pg_var_type("StructAggState");
1034 393 : StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1035 393 : StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1036 :
1158 1037 393 : AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1844 1038 393 : }
1039 :
1040 : /*
1041 : * Split a symbol into module / function parts. If the function is in the
1042 : * main binary (or an external library) *modname will be NULL.
1043 : */
1044 : void
1045 17623 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1046 : {
1047 17623 : *modname = NULL;
1048 17623 : *funcname = NULL;
1049 :
1050 : /*
1051 : * Module function names are pgextern.$module.$funcname
1052 : */
1053 17623 : if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1054 : {
1055 : /*
1056 : * Symbol names cannot contain a ., therefore we can split based on
1057 : * first and last occurrence of one.
1058 : */
1059 42 : *funcname = rindex(name, '.');
1060 42 : (*funcname)++; /* jump over . */
1061 :
1062 84 : *modname = pnstrdup(name + strlen("pgextern."),
1063 42 : *funcname - name - strlen("pgextern.") - 1);
1064 42 : Assert(funcname);
1065 :
1066 42 : *funcname = pstrdup(*funcname);
1067 : }
1068 : else
1069 : {
1070 17581 : *modname = NULL;
1071 17581 : *funcname = pstrdup(name);
1072 : }
1073 17623 : }
1074 :
1075 : /*
1076 : * Attempt to resolve symbol, so LLVM can emit a reference to it.
1077 : */
1078 : static uint64_t
1079 8 : llvm_resolve_symbol(const char *symname, void *ctx)
1080 : {
1081 : uintptr_t addr;
1082 : char *funcname;
1083 : char *modname;
1084 :
1085 : /*
1086 : * macOS prefixes all object level symbols with an underscore. But neither
1087 : * dlsym() nor PG's inliner expect that. So undo.
1088 : */
1089 : #if defined(__darwin__)
1090 : if (symname[0] != '_')
1091 : elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1092 : symname++;
1093 : #endif
1094 :
1095 8 : llvm_split_symbol_name(symname, &modname, &funcname);
1096 :
1097 : /* functions that aren't resolved to names shouldn't ever get here */
1098 8 : Assert(funcname);
1099 :
1100 8 : if (modname)
1101 8 : addr = (uintptr_t) load_external_function(modname, funcname,
1102 : true, NULL);
1103 : else
1844 andres 1104 UBC 0 : addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1105 :
1844 andres 1106 CBC 8 : pfree(funcname);
1107 8 : if (modname)
1108 8 : pfree(modname);
1109 :
1110 : /* let LLVM will error out - should never happen */
1111 8 : if (!addr)
1844 andres 1112 UBC 0 : elog(WARNING, "failed to resolve name %s", symname);
1113 :
1844 andres 1114 CBC 8 : return (uint64_t) addr;
1115 : }
1116 :
1117 : #if LLVM_VERSION_MAJOR > 11
1118 :
1119 : static LLVMErrorRef
881 1120 8 : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1121 : LLVMOrcLookupStateRef * LookupState, LLVMOrcLookupKind Kind,
1122 : LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1123 : LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1124 : {
1125 : #if LLVM_VERSION_MAJOR > 14
1126 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1127 : #else
1128 8 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1129 : #endif
1130 : LLVMErrorRef error;
1131 : LLVMOrcMaterializationUnitRef mu;
1132 :
1133 16 : for (int i = 0; i < LookupSetSize; i++)
1134 : {
1135 8 : const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1136 :
1137 : #if LLVM_VERSION_MAJOR > 12
653 tmunro 1138 8 : LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1139 : #endif
881 andres 1140 8 : symbols[i].Name = LookupSet[i].Name;
1141 8 : symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1142 8 : symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1143 : }
1144 :
1145 8 : mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1146 8 : error = LLVMOrcJITDylibDefine(JD, mu);
1147 8 : if (error != LLVMErrorSuccess)
881 andres 1148 UBC 0 : LLVMOrcDisposeMaterializationUnit(mu);
1149 :
881 andres 1150 CBC 8 : pfree(symbols);
1151 :
1152 8 : return error;
1153 : }
1154 :
1155 : /*
1156 : * We cannot throw errors through LLVM (without causing a FATAL at least), so
1157 : * just use WARNING here. That's OK anyway, as the error is also reported at
1158 : * the top level action (with less detail) and there might be multiple
1159 : * invocations of errors with details.
1160 : *
1161 : * This doesn't really happen during normal operation, but in cases like
1162 : * symbol resolution breakage. So just using elog(WARNING) is fine.
1163 : */
1164 : static void
881 andres 1165 UBC 0 : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1166 : {
1167 0 : elog(WARNING, "error during JITing: %s",
1168 : llvm_error_message(error));
1169 0 : }
1170 :
1171 : /*
1172 : * Create our own object layer, so we can add event listeners.
1173 : */
1174 : static LLVMOrcObjectLayerRef
881 andres 1175 CBC 786 : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1176 : {
1177 : LLVMOrcObjectLayerRef objlayer =
1178 786 : LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1179 :
1180 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1181 786 : if (jit_debugging_support)
1182 : {
881 andres 1183 UBC 0 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1184 :
1185 0 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1186 : }
1187 : #endif
1188 :
1189 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
881 andres 1190 CBC 786 : if (jit_profiling_support)
1191 : {
881 andres 1192 UBC 0 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1193 :
1194 0 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1195 : }
1196 : #endif
1197 :
881 andres 1198 CBC 786 : return objlayer;
1199 : }
1200 :
1201 : /*
1202 : * Create LLJIT instance, using the passed in target machine. Note that the
1203 : * target machine afterwards is owned by the LLJIT instance.
1204 : */
1205 : static LLVMOrcLLJITRef
1206 786 : llvm_create_jit_instance(LLVMTargetMachineRef tm)
1207 : {
1208 : LLVMOrcLLJITRef lljit;
1209 : LLVMOrcJITTargetMachineBuilderRef tm_builder;
1210 : LLVMOrcLLJITBuilderRef lljit_builder;
1211 : LLVMErrorRef error;
1212 : LLVMOrcDefinitionGeneratorRef main_gen;
1213 : LLVMOrcDefinitionGeneratorRef ref_gen;
1214 :
1215 786 : lljit_builder = LLVMOrcCreateLLJITBuilder();
1216 786 : tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1217 786 : LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1218 :
1219 786 : LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1220 : llvm_create_object_layer,
1221 : NULL);
1222 :
1223 786 : error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1224 786 : if (error)
881 andres 1225 UBC 0 : elog(ERROR, "failed to create lljit instance: %s",
1226 : llvm_error_message(error));
1227 :
881 andres 1228 CBC 786 : LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1229 : llvm_log_jit_error, NULL);
1230 :
1231 : /*
1232 : * Symbol resolution support for symbols in the postgres binary /
1233 : * libraries already loaded.
1234 : */
1235 786 : error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1236 786 : LLVMOrcLLJITGetGlobalPrefix(lljit),
1237 : 0, NULL);
1238 786 : if (error)
881 andres 1239 UBC 0 : elog(ERROR, "failed to create generator: %s",
1240 : llvm_error_message(error));
881 andres 1241 CBC 786 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1242 :
1243 : /*
1244 : * Symbol resolution support for "special" functions, e.g. a call into an
1245 : * SQL callable function.
1246 : */
1247 : #if LLVM_VERSION_MAJOR > 14
1248 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1249 : #else
1250 786 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1251 : #endif
1252 786 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1253 :
1254 786 : return lljit;
1255 : }
1256 :
1257 : static char *
881 andres 1258 UBC 0 : llvm_error_message(LLVMErrorRef error)
1259 : {
1260 0 : char *orig = LLVMGetErrorMessage(error);
1261 0 : char *msg = pstrdup(orig);
1262 :
1263 0 : LLVMDisposeErrorMessage(orig);
1264 :
1265 0 : return msg;
1266 : }
1267 :
1268 : #endif /* LLVM_VERSION_MAJOR > 11 */
|
