Age Owner TLA Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit_inline.cpp
4 : * Cross module inlining suitable for postgres' JIT
5 : *
6 : * The inliner iterates over external functions referenced from the passed
7 : * module and attempts to inline those. It does so by utilizing pre-built
8 : * indexes over both postgres core code and extension modules. When a match
9 : * for an external function is found - not guaranteed! - the index will then
10 : * be used to judge their instruction count / inline worthiness. After doing
11 : * so for all external functions, all the referenced functions (and
12 : * prerequisites) will be imported.
13 : *
14 : * Copyright (c) 2016-2023, PostgreSQL Global Development Group
15 : *
16 : * IDENTIFICATION
17 : * src/backend/lib/llvmjit/llvmjit_inline.cpp
18 : *
19 : *-------------------------------------------------------------------------
20 : */
21 :
22 : extern "C"
23 : {
24 : #include "postgres.h"
25 : }
26 :
27 : #include "jit/llvmjit.h"
28 :
29 : extern "C"
30 : {
31 : #include <fcntl.h>
32 : #include <sys/mman.h>
33 : #include <sys/stat.h>
34 : #include <sys/types.h>
35 : #include <unistd.h>
36 :
37 : #include "common/string.h"
38 : #include "miscadmin.h"
39 : #include "storage/fd.h"
40 : }
41 :
42 : #include <llvm-c/Core.h>
43 : #include <llvm-c/BitReader.h>
44 :
45 : /* Avoid macro clash with LLVM's C++ headers */
46 : #undef Min
47 :
48 : #include <llvm/ADT/SetVector.h>
49 : #include <llvm/ADT/StringSet.h>
50 : #include <llvm/ADT/StringMap.h>
51 : #include <llvm/Analysis/ModuleSummaryAnalysis.h>
52 : #if LLVM_VERSION_MAJOR > 3
53 : #include <llvm/Bitcode/BitcodeReader.h>
54 : #else
55 : #include <llvm/Bitcode/ReaderWriter.h>
56 : #include <llvm/Support/Error.h>
57 : #endif
58 : #include <llvm/IR/Attributes.h>
59 : #include <llvm/IR/DebugInfo.h>
60 : #include <llvm/IR/IntrinsicInst.h>
61 : #include <llvm/IR/IRBuilder.h>
62 : #include <llvm/IR/ModuleSummaryIndex.h>
63 : #include <llvm/Linker/IRMover.h>
64 : #include <llvm/Support/ManagedStatic.h>
65 : #include <llvm/Support/MemoryBuffer.h>
66 :
67 :
68 : /*
69 : * Type used to represent modules InlineWorkListItem's subject is searched for
70 : * in.
71 : */
72 : typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
73 :
74 : /*
75 : * Item in queue of to-be-checked symbols and corresponding queue.
76 : */
77 : typedef struct InlineWorkListItem
78 : {
79 : llvm::StringRef symbolName;
80 : llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
81 : } InlineWorkListItem;
82 : typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
83 :
84 : /*
85 : * Information about symbols processed during inlining. Used to prevent
86 : * repeated searches and provide additional information.
87 : */
88 : typedef struct FunctionInlineState
89 : {
90 : int costLimit;
91 : bool processed;
92 : bool inlined;
93 : bool allowReconsidering;
94 : } FunctionInlineState;
95 : typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
96 :
97 : /*
98 : * Map of modules that should be inlined, with a list of the to-be inlined
99 : * symbols.
100 : */
101 : typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
102 :
103 :
104 : const float inline_cost_decay_factor = 0.5;
105 : const int inline_initial_cost = 150;
106 :
107 : /*
108 : * These are managed statics so LLVM knows to deallocate them during an
109 : * LLVMShutdown(), rather than after (which'd cause crashes).
110 : */
111 : typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
112 : llvm::ManagedStatic<ModuleCache> module_cache;
113 : typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
114 : llvm::ManagedStatic<SummaryCache> summary_cache;
115 :
116 :
117 : static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(llvm::Module *mod);
118 : static void llvm_execute_inline_plan(llvm::Module *mod,
119 : ImportMapTy *globalsToInline);
120 :
121 : static llvm::Module* load_module_cached(llvm::StringRef modPath);
122 : static std::unique_ptr<llvm::Module> load_module(llvm::StringRef Identifier);
123 : static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
124 :
125 :
126 : static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
127 : llvm::Function *F,
128 : llvm::StringRef Name);
129 :
130 : static bool function_inlinable(llvm::Function &F,
131 : int threshold,
132 : FunctionInlineStates &functionState,
133 : InlineWorkList &worklist,
134 : InlineSearchPath &searchpath,
135 : llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
136 : int &running_instcount,
137 : llvm::StringSet<> &importVars);
138 : static void function_references(llvm::Function &F,
139 : int &running_instcount,
140 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
141 : llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
142 :
143 : static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
144 : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
145 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
146 :
147 : /* verbose debugging for inliner development */
148 : /* #define INLINE_DEBUG */
149 : #ifdef INLINE_DEBUG
150 : #define ilog elog
151 : #else
152 : #define ilog(...) (void) 0
153 : #endif
154 :
155 : /*
156 : * Perform inlining of external function references in M based on a simple
157 : * cost based analysis.
158 : */
159 : void
1838 andres 160 CBC 500 : llvm_inline(LLVMModuleRef M)
161 : {
162 500 : llvm::Module *mod = llvm::unwrap(M);
163 :
164 500 : std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(mod);
165 500 : if (!globalsToInline)
1838 andres 166 UBC 0 : return;
1838 andres 167 CBC 500 : llvm_execute_inline_plan(mod, globalsToInline.get());
168 500 : }
169 :
170 : /*
171 : * Build information necessary for inlining external function references in
172 : * mod.
173 : */
174 : static std::unique_ptr<ImportMapTy>
175 500 : llvm_build_inline_plan(llvm::Module *mod)
176 : {
1323 tmunro 177 500 : std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
1838 andres 178 500 : FunctionInlineStates functionStates;
179 500 : InlineWorkList worklist;
180 :
181 500 : InlineSearchPath defaultSearchPath;
182 :
183 : /* attempt to add module to search path */
184 500 : add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
185 : /* if postgres isn't available, no point continuing */
186 500 : if (defaultSearchPath.empty())
1838 andres 187 UBC 0 : return nullptr;
188 :
189 : /*
190 : * Start inlining with current references to external functions by putting
191 : * them on the inlining worklist. If, during inlining of those, new extern
192 : * functions need to be inlined, they'll also be put there, with a lower
193 : * priority.
194 : */
1838 andres 195 CBC 7667 : for (const llvm::Function &funcDecl : mod->functions())
196 : {
197 7167 : InlineWorkListItem item = {};
198 7167 : FunctionInlineState inlineState = {};
199 :
200 : /* already has a definition */
201 7167 : if (!funcDecl.isDeclaration())
202 4895 : continue;
203 :
204 : /* llvm provides implementation */
205 2272 : if (funcDecl.isIntrinsic())
206 408 : continue;
207 :
208 1864 : item.symbolName = funcDecl.getName();
209 1864 : item.searchpath = defaultSearchPath;
210 1864 : worklist.push_back(item);
211 1864 : inlineState.costLimit = inline_initial_cost;
212 1864 : inlineState.processed = false;
213 1864 : inlineState.inlined = false;
214 1864 : inlineState.allowReconsidering = false;
215 1864 : functionStates[funcDecl.getName()] = inlineState;
216 7167 : }
217 :
218 : /*
219 : * Iterate over pending worklist items, look them up in index, check
220 : * whether they should be inlined.
221 : */
222 11811 : while (!worklist.empty())
223 : {
224 11311 : InlineWorkListItem item = worklist.pop_back_val();
225 11311 : llvm::StringRef symbolName = item.symbolName;
226 : char *cmodname;
227 : char *cfuncname;
228 11311 : FunctionInlineState &inlineState = functionStates[symbolName];
229 : llvm::GlobalValue::GUID funcGUID;
230 :
231 11311 : llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
232 :
233 11311 : funcGUID = llvm::GlobalValue::getGUID(cfuncname);
234 :
235 : /* already processed */
236 11311 : if (inlineState.processed)
1838 andres 237 UBC 0 : continue;
238 :
239 :
1838 andres 240 CBC 11311 : if (cmodname)
241 18 : add_module_to_inline_search_path(item.searchpath, cmodname);
242 :
243 : /*
244 : * Iterate over all known definitions of function, via the index. Then
245 : * look up module(s), check if function actually is defined (there
246 : * could be hash conflicts).
247 : */
248 17481 : for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
249 : {
250 : const llvm::FunctionSummary *fs;
251 9310 : llvm::StringRef modPath = gvs->modulePath();
252 : llvm::Module *defMod;
253 : llvm::Function *funcDef;
254 :
255 9310 : fs = llvm::cast<llvm::FunctionSummary>(gvs);
256 :
257 : #if LLVM_VERSION_MAJOR > 3
258 9310 : if (gvs->notEligibleToImport())
259 : {
260 : ilog(DEBUG1, "ineligibile to import %s due to summary",
261 : symbolName.data());
262 5176 : continue;
263 : }
264 : #endif
265 :
266 9310 : if ((int) fs->instCount() > inlineState.costLimit)
267 : {
268 : ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
269 : symbolName.data(), fs->instCount(), inlineState.costLimit);
270 4228 : inlineState.allowReconsidering = true;
271 4228 : continue;
272 : }
273 :
274 5082 : defMod = load_module_cached(modPath);
275 5082 : if (defMod->materializeMetadata())
1838 andres 276 UBC 0 : elog(FATAL, "failed to materialize metadata");
277 :
1838 andres 278 CBC 5082 : funcDef = defMod->getFunction(cfuncname);
279 :
280 : /*
281 : * This can happen e.g. in case of a hash collision of the
282 : * function's name.
283 : */
284 5082 : if (!funcDef)
1838 andres 285 UBC 0 : continue;
286 :
1838 andres 287 CBC 5082 : if (funcDef->materialize())
1838 andres 288 UBC 0 : elog(FATAL, "failed to materialize metadata");
289 :
1838 andres 290 CBC 5082 : Assert(!funcDef->isDeclaration());
291 5082 : Assert(funcDef->hasExternalLinkage());
292 :
293 5082 : llvm::StringSet<> importVars;
294 5082 : llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
295 5082 : int running_instcount = 0;
296 :
297 : /*
298 : * Check whether function, and objects it depends on, are
299 : * inlinable.
300 : */
301 5082 : if (function_inlinable(*funcDef,
302 : inlineState.costLimit,
303 : functionStates,
304 : worklist,
305 : item.searchpath,
306 : visitedFunctions,
307 : running_instcount,
308 : importVars))
309 : {
310 : /*
311 : * Check whether function and all its dependencies are too
312 : * big. Dependencies already counted for other functions that
313 : * will get inlined are not counted again. While this make
314 : * things somewhat order dependent, I can't quite see a point
315 : * in a different behaviour.
316 : */
317 4088 : if (running_instcount > inlineState.costLimit)
318 : {
319 : ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
320 : symbolName.data(), running_instcount, inlineState.costLimit);
321 948 : inlineState.allowReconsidering = true;
322 948 : continue;
323 : }
324 :
325 : ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
326 : symbolName.data(), running_instcount, fs->instCount());
327 :
328 : /* import referenced function itself */
329 3140 : importVars.insert(symbolName);
330 :
331 : {
332 3140 : llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
333 9570 : for (auto& importVar : importVars)
334 6430 : modGlobalsToInline.insert(importVar.first());
335 3140 : Assert(modGlobalsToInline.size() > 0);
336 : }
337 :
338 : /* mark function as inlined */
339 3140 : inlineState.inlined = true;
340 :
341 : /*
342 : * Found definition to inline, don't look for further
343 : * potential definitions.
344 : */
345 3140 : break;
346 : }
347 : else
348 : {
349 : ilog(DEBUG1, "had to skip inlining %s",
350 : symbolName.data());
351 :
352 : /* It's possible there's another definition that's inlinable. */
353 : }
354 20481 : }
355 :
356 : /*
357 : * Signal that we're done with symbol, whether successful (inlined =
358 : * true above) or not.
359 : */
360 11311 : inlineState.processed = true;
361 11311 : }
362 :
363 500 : return globalsToInline;
364 500 : }
365 :
366 : /*
367 : * Perform the actual inlining of external functions (and their dependencies)
368 : * into mod.
369 : */
370 : static void
371 500 : llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
372 : {
373 500 : llvm::IRMover Mover(*mod);
374 :
375 3099 : for (const auto& toInline : *globalsToInline)
376 : {
377 2599 : const llvm::StringRef& modPath = toInline.first();
378 2599 : const llvm::StringSet<>& modGlobalsToInline = toInline.second;
379 2599 : llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
380 :
381 2599 : Assert(module_cache->count(modPath));
382 2599 : std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
383 2599 : module_cache->erase(modPath);
384 :
385 2599 : if (modGlobalsToInline.empty())
1838 andres 386 UBC 0 : continue;
387 :
1838 andres 388 CBC 8903 : for (auto &glob: modGlobalsToInline)
389 : {
390 6304 : llvm::StringRef SymbolName = glob.first();
391 : char *modname;
392 : char *funcname;
393 :
394 6304 : llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
395 :
396 6304 : llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
397 :
398 6304 : if (!valueToImport)
1838 andres 399 UBC 0 : elog(FATAL, "didn't refind value %s to import", SymbolName.data());
400 :
401 : /*
402 : * For functions (global vars are only inlined if already static),
403 : * mark imported variables as being clones from other
404 : * functions. That a) avoids symbol conflicts b) allows the
405 : * optimizer to perform inlining.
406 : */
1838 andres 407 CBC 6304 : if (llvm::isa<llvm::Function>(valueToImport))
408 : {
409 3177 : llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
410 : typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
411 :
412 : /*
413 : * Per-function info isn't necessarily stripped yet, as the
414 : * module is lazy-loaded when stripped above.
415 : */
416 3177 : llvm::stripDebugInfo(*F);
417 :
418 : /*
419 : * If the to-be-imported function is one referenced including
420 : * its module name, create a tiny inline function that just
421 : * forwards the call. One might think a GlobalAlias would do
422 : * the trick, but a) IRMover doesn't override a declaration
423 : * with an alias pointing to a definition (instead renaming
424 : * it), b) Aliases can't be AvailableExternally.
425 : */
426 3177 : if (modname)
427 : {
428 : llvm::Function *AF;
429 :
430 16 : AF = create_redirection_function(importMod, F, SymbolName);
431 :
432 16 : GlobalsToImport.insert(AF);
433 16 : llvm::stripDebugInfo(*AF);
434 : }
435 :
436 3177 : if (valueToImport->hasExternalLinkage())
437 : {
438 3140 : valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
439 : }
440 : }
441 :
442 6304 : GlobalsToImport.insert(valueToImport);
443 : ilog(DEBUG1, "performing import of %s %s",
444 : modPath.data(), SymbolName.data());
445 :
446 : }
447 :
448 : #if LLVM_VERSION_MAJOR > 4
449 : #define IRMOVE_PARAMS , /*IsPerformingImport=*/false
450 : #elif LLVM_VERSION_MAJOR > 3
451 : #define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
452 : #else
453 : #define IRMOVE_PARAMS
454 : #endif
455 2599 : if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
456 2599 : [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
457 : IRMOVE_PARAMS))
1838 andres 458 UBC 0 : elog(FATAL, "function import failed with linker error");
1838 andres 459 CBC 2599 : }
460 500 : }
461 :
462 : /*
463 : * Return a module identified by modPath, caching it in memory.
464 : *
465 : * Note that such a module may *not* be modified without copying, otherwise
466 : * the cache state would get corrupted.
467 : */
468 : static llvm::Module*
469 5082 : load_module_cached(llvm::StringRef modPath)
470 : {
471 5082 : auto it = module_cache->find(modPath);
472 5082 : if (it == module_cache->end())
473 : {
474 5500 : it = module_cache->insert(
475 5500 : std::make_pair(modPath, load_module(modPath))).first;
476 : }
477 :
478 5082 : return it->second.get();
479 : }
480 :
481 : static std::unique_ptr<llvm::Module>
482 2750 : load_module(llvm::StringRef Identifier)
483 : {
484 : LLVMMemoryBufferRef buf;
485 : LLVMModuleRef mod;
486 : char path[MAXPGPATH];
487 : char *msg;
488 :
489 2750 : snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
490 :
491 2750 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1838 andres 492 UBC 0 : elog(FATAL, "failed to open bitcode file \"%s\": %s",
493 : path, msg);
1838 andres 494 CBC 2750 : if (LLVMGetBitcodeModuleInContext2(LLVMGetGlobalContext(), buf, &mod))
1838 andres 495 UBC 0 : elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
496 :
497 : /*
498 : * Currently there's no use in more detailed debug info for JITed
499 : * code. Until that changes, not much point in wasting memory and cycles
500 : * on processing debuginfo.
501 : */
1838 andres 502 CBC 2750 : llvm::StripDebugInfo(*llvm::unwrap(mod));
503 :
504 5500 : return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
505 : }
506 :
507 : /*
508 : * Compute list of referenced variables, functions and the instruction count
509 : * for a function.
510 : */
511 : static void
512 7587 : function_references(llvm::Function &F,
513 : int &running_instcount,
514 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
515 : llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
516 : {
517 7587 : llvm::SmallPtrSet<const llvm::User *, 32> Visited;
518 :
519 217383 : for (llvm::BasicBlock &BB : F)
520 : {
521 1066460 : for (llvm::Instruction &I : BB)
522 : {
523 480781 : if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
1838 andres 524 UBC 0 : continue;
525 :
1838 andres 526 CBC 480781 : llvm::SmallVector<llvm::User *, 8> Worklist;
527 480781 : Worklist.push_back(&I);
528 :
529 480781 : running_instcount++;
530 :
531 1780390 : while (!Worklist.empty()) {
532 1299609 : llvm::User *U = Worklist.pop_back_val();
533 :
534 : /* visited before */
535 1299609 : if (!Visited.insert(U).second)
536 670912 : continue;
537 :
538 1734789 : for (auto &OI : U->operands()) {
539 1106092 : llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
540 1106092 : if (!Operand)
541 320099 : continue;
542 907513 : if (llvm::isa<llvm::BlockAddress>(Operand))
1838 andres 543 UBC 0 : continue;
1838 andres 544 CBC 907513 : if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
545 35715 : referencedVars.insert(GV);
546 35715 : if (GV->hasInitializer())
547 32835 : Worklist.push_back(GV->getInitializer());
548 35715 : continue;
549 : }
550 871798 : if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
551 85805 : referencedFunctions.insert(CF);
552 85805 : continue;
553 : }
554 785993 : Worklist.push_back(Operand);
555 : }
556 : }
557 480781 : }
558 : }
559 7587 : }
560 :
561 : /*
562 : * Check whether function F is inlinable and, if so, what globals need to be
563 : * imported.
564 : *
565 : * References to external functions from, potentially recursively, inlined
566 : * functions are added to the passed in worklist.
567 : */
568 : static bool
569 7593 : function_inlinable(llvm::Function &F,
570 : int threshold,
571 : FunctionInlineStates &functionStates,
572 : InlineWorkList &worklist,
573 : InlineSearchPath &searchpath,
574 : llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
575 : int &running_instcount,
576 : llvm::StringSet<> &importVars)
577 : {
578 7593 : int subThreshold = threshold * inline_cost_decay_factor;
579 7593 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
580 7593 : llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
581 :
582 : /* can't rely on what may be inlined */
583 7593 : if (F.isInterposable())
1838 andres 584 UBC 0 : return false;
585 :
586 : /*
587 : * Can't rely on function being present. Alternatively we could create a
588 : * static version of these functions?
589 : */
1838 andres 590 CBC 7593 : if (F.hasAvailableExternallyLinkage())
1838 andres 591 UBC 0 : return false;
592 :
593 : ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
594 :
1838 andres 595 CBC 7593 : if (F.materialize())
1838 andres 596 UBC 0 : elog(FATAL, "failed to materialize metadata");
597 :
598 : #if LLVM_VERSION_MAJOR < 14
599 : #define hasFnAttr hasFnAttribute
600 : #endif
601 :
559 tmunro 602 CBC 7593 : if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
603 : {
604 : ilog(DEBUG1, "ineligibile to import %s due to noinline",
605 : F.getName().data());
1719 andres 606 6 : return false;
607 : }
608 :
1838 609 7587 : function_references(F, running_instcount, referencedVars, referencedFunctions);
610 :
611 33943 : for (llvm::GlobalVariable* rv: referencedVars)
612 : {
613 27344 : if (rv->materialize())
1838 andres 614 UBC 0 : elog(FATAL, "failed to materialize metadata");
615 :
616 : /*
617 : * Don't inline functions that access thread local variables. That
618 : * doesn't work on current LLVM releases (but might in future).
619 : */
626 tmunro 620 CBC 27344 : if (rv->isThreadLocal())
621 : {
622 : ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
623 : F.getName().data(), rv->getName().data());
624 988 : return false;
625 : }
626 :
627 : /*
628 : * Never want to inline externally visible vars, cheap enough to
629 : * reference.
630 : */
1838 andres 631 27344 : if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
632 887 : continue;
633 :
634 : /*
635 : * If variable is file-local, we need to inline it, to be able to
636 : * inline the function itself. Can't do that if the variable can be
637 : * modified, because they'd obviously get out of sync.
638 : *
639 : * XXX: Currently not a problem, but there'd be problems with
640 : * nontrivial initializers if they were allowed for postgres.
641 : */
642 26457 : if (!rv->isConstant())
643 : {
644 : ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
645 : F.getName().data(), rv->getName().data());
646 988 : return false;
647 : }
648 :
649 : ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
650 : rv->getName().data(), (int)rv->getLinkage());
651 :
652 25469 : importVars.insert(rv->getName());
653 : /* small cost attributed to each cloned global */
654 25469 : running_instcount += 5;
655 : }
656 :
657 6599 : visitedFunctions.insert(&F);
658 :
659 : /*
660 : * Check referenced functions. Check whether used static ones are
661 : * inlinable, and remember external ones for inlining.
662 : */
663 44276 : for (llvm::Function* referencedFunction: referencedFunctions)
664 : {
665 37715 : llvm::StringSet<> recImportVars;
666 :
667 37715 : if (referencedFunction->materialize())
1838 andres 668 UBC 0 : elog(FATAL, "failed to materialize metadata");
669 :
1838 andres 670 CBC 37715 : if (referencedFunction->isIntrinsic())
671 3591 : continue;
672 :
673 : /* if already visited skip, otherwise remember */
674 34124 : if (!visitedFunctions.insert(referencedFunction).second)
675 12565 : continue;
676 :
677 : /*
678 : * We don't inline external functions directly here, instead we put
679 : * them on the worklist if appropriate and check them from
680 : * llvm_build_inline_plan().
681 : */
682 21559 : if (referencedFunction->hasExternalLinkage())
683 : {
684 19048 : llvm::StringRef funcName = referencedFunction->getName();
685 :
686 : /*
687 : * Don't bother checking for inlining if remaining cost budget is
688 : * very small.
689 : */
690 19048 : if (subThreshold < 5)
691 2186 : continue;
692 :
693 16862 : auto it = functionStates.find(funcName);
694 16862 : if (it == functionStates.end())
695 : {
696 : FunctionInlineState inlineState;
697 :
698 9078 : inlineState.costLimit = subThreshold;
699 9078 : inlineState.processed = false;
700 9078 : inlineState.inlined = false;
701 9078 : inlineState.allowReconsidering = false;
702 :
703 9078 : functionStates[funcName] = inlineState;
704 9078 : worklist.push_back({funcName, searchpath});
705 :
706 : ilog(DEBUG1,
707 : "considering extern function %s at %d for inlining",
708 : funcName.data(), subThreshold);
709 : }
710 7784 : else if (!it->second.inlined &&
711 13205 : (!it->second.processed || it->second.allowReconsidering) &&
712 13205 : it->second.costLimit < subThreshold)
713 : {
714 : /*
715 : * Update inlining threshold if higher. Need to re-queue
716 : * to be processed if already processed with lower
717 : * threshold.
718 : */
719 407 : if (it->second.processed)
720 : {
721 : ilog(DEBUG1,
722 : "reconsidering extern function %s at %d for inlining, increasing from %d",
723 : funcName.data(), subThreshold, it->second.costLimit);
724 :
725 369 : it->second.processed = false;
726 369 : it->second.allowReconsidering = false;
727 369 : worklist.push_back({funcName, searchpath});
728 : }
729 407 : it->second.costLimit = subThreshold;
730 : }
731 16862 : continue;
732 16862 : }
733 :
734 : /* can't rely on what may be inlined */
735 2511 : if (referencedFunction->isInterposable())
1838 andres 736 UBC 0 : return false;
737 :
1838 andres 738 CBC 2511 : if (!function_inlinable(*referencedFunction,
739 : subThreshold,
740 : functionStates,
741 : worklist,
742 : searchpath,
743 : visitedFunctions,
744 : running_instcount,
745 : recImportVars))
746 : {
747 : ilog(DEBUG1,
748 : "cannot inline %s due to required function %s not being inlinable",
749 : F.getName().data(), referencedFunction->getName().data());
750 38 : return false;
751 : }
752 :
753 : /* import referenced function itself */
754 2473 : importVars.insert(referencedFunction->getName());
755 :
756 : /* import referenced function and its dependents */
757 52416 : for (auto& recImportVar : recImportVars)
758 49943 : importVars.insert(recImportVar.first());
759 37715 : }
760 :
761 6561 : return true;
762 17040 : }
763 :
764 : /*
765 : * Attempt to load module summary located at path. Return empty pointer when
766 : * loading fails.
767 : */
768 : static std::unique_ptr<llvm::ModuleSummaryIndex>
769 249 : llvm_load_summary(llvm::StringRef path)
770 : {
771 : llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
772 249 : llvm::MemoryBuffer::getFile(path);
773 :
774 249 : if (std::error_code EC = MBOrErr.getError())
775 : {
776 : ilog(DEBUG1, "failed to open %s: %s", path.data(),
777 : EC.message().c_str());
778 : }
779 : else
780 : {
781 249 : llvm::MemoryBufferRef ref(*MBOrErr.get().get());
782 :
783 : #if LLVM_VERSION_MAJOR > 3
784 : llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
785 249 : llvm::getModuleSummaryIndex(ref);
786 249 : if (IndexOrErr)
787 249 : return std::move(IndexOrErr.get());
1838 andres 788 UBC 0 : elog(FATAL, "failed to load summary \"%s\": %s",
789 : path.data(),
790 : toString(IndexOrErr.takeError()).c_str());
791 : #else
792 : llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
793 : llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {});
794 : if (IndexOrErr)
795 : return std::move(IndexOrErr.get());
796 : elog(FATAL, "failed to load summary \"%s\": %s",
797 : path.data(),
798 : IndexOrErr.getError().message().c_str());
799 : #endif
1838 andres 800 CBC 249 : }
1838 andres 801 UBC 0 : return nullptr;
1838 andres 802 CBC 249 : }
803 :
804 : /*
805 : * Attempt to add modpath to the search path.
806 : */
807 : static void
808 518 : add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
809 : {
810 : /* only extension in libdir are candidates for inlining for now */
811 518 : if (!modpath.startswith("$libdir/"))
1838 andres 812 UBC 0 : return;
813 :
814 : /* if there's no match, attempt to load */
1838 andres 815 CBC 518 : auto it = summary_cache->find(modpath);
816 518 : if (it == summary_cache->end())
817 : {
818 249 : std::string path(modpath);
819 249 : path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
820 249 : path += ".index.bc";
821 249 : (*summary_cache)[modpath] = llvm_load_summary(path);
822 249 : it = summary_cache->find(modpath);
823 249 : }
824 :
825 518 : Assert(it != summary_cache->end());
826 :
827 : /* if the entry isn't NULL, it's validly loaded */
828 518 : if (it->second)
829 518 : searchpath.push_back(it->second.get());
830 : }
831 :
832 : /*
833 : * Search for all references for functions hashing to guid in the search path,
834 : * and return them in search path order.
835 : */
836 : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
837 11311 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
838 : {
839 11311 : llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
840 :
841 23298 : for (auto index : path)
842 : {
843 : #if LLVM_VERSION_MAJOR > 4
844 11987 : llvm::ValueInfo funcVI = index->getValueInfo(guid);
845 :
846 : /* if index doesn't know function, we don't have a body, continue */
847 11987 : if (funcVI)
848 18620 : for (auto &gv : funcVI.getSummaryList())
849 9310 : matches.push_back(gv.get());
850 : #else
851 : const llvm::const_gvsummary_iterator &I =
852 : index->findGlobalValueSummaryList(guid);
853 : if (I != index->end())
854 : {
855 : for (auto &gv : I->second)
856 : matches.push_back(gv.get());
857 : }
858 : #endif
859 : }
860 :
861 11311 : return matches;
1838 andres 862 UBC 0 : }
863 :
864 : /*
865 : * Create inline wrapper with the name Name, redirecting the call to F.
866 : */
867 : static llvm::Function*
1838 andres 868 CBC 16 : create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
869 : llvm::Function *F,
870 : llvm::StringRef Name)
871 : {
872 : typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
873 :
874 16 : llvm::LLVMContext &Context = F->getContext();
875 16 : llvm::IRBuilder<> Builder(Context);
876 : llvm::Function *AF;
877 : llvm::BasicBlock *BB;
878 : llvm::CallInst *fwdcall;
879 : #if LLVM_VERSION_MAJOR < 14
880 : llvm::Attribute inlineAttribute;
881 : #endif
882 :
883 16 : AF = llvm::Function::Create(F->getFunctionType(),
884 : LinkageTypes::AvailableExternallyLinkage,
885 : Name, importMod.get());
886 16 : BB = llvm::BasicBlock::Create(Context, "entry", AF);
887 :
888 16 : Builder.SetInsertPoint(BB);
889 16 : fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
890 : #if LLVM_VERSION_MAJOR < 14
891 : inlineAttribute = llvm::Attribute::get(Context,
892 : llvm::Attribute::AlwaysInline);
893 : fwdcall->addAttribute(~0U, inlineAttribute);
894 : #else
559 tmunro 895 16 : fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
896 : #endif
1838 andres 897 16 : Builder.CreateRet(fwdcall);
898 :
899 16 : return AF;
900 16 : }
|
