1 /*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "bcc/Assert.h"
18 #include "bcc/Renderscript/RSTransforms.h"
19
20 #include <cstdlib>
21 #include <functional>
22
23 #include <llvm/IR/DerivedTypes.h>
24 #include <llvm/IR/Function.h>
25 #include <llvm/IR/Instructions.h>
26 #include <llvm/IR/IRBuilder.h>
27 #include <llvm/IR/MDBuilder.h>
28 #include <llvm/IR/Module.h>
29 #include <llvm/Pass.h>
30 #include <llvm/Support/raw_ostream.h>
31 #include <llvm/IR/DataLayout.h>
32 #include <llvm/IR/Function.h>
33 #include <llvm/IR/Type.h>
34 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
35
36 #include "bcc/Config/Config.h"
37 #include "bcc/Support/Log.h"
38
39 #include "bcinfo/MetadataExtractor.h"
40
41 #define NUM_EXPANDED_FUNCTION_PARAMS 4
42
43 using namespace bcc;
44
45 namespace {
46
47 static const bool gEnableRsTbaa = true;
48
49 /* RSForEachExpandPass - This pass operates on functions that are able to be
50 * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the
51 * ForEach-able function to be invoked over the appropriate data cells of the
52 * input/output allocations (adjusting other relevant parameters as we go). We
53 * support doing this for any ForEach-able compute kernels. The new function
54 * name is the original function name followed by ".expand". Note that we
55 * still generate code for the original function.
56 */
57 class RSForEachExpandPass : public llvm::ModulePass {
58 public:
59 static char ID;
60
61 private:
62 static const size_t RS_KERNEL_INPUT_LIMIT = 8; // see frameworks/base/libs/rs/cpu_ref/rsCpuCoreRuntime.h
63
64 enum RsLaunchDimensionsField {
65 RsLaunchDimensionsFieldX,
66 RsLaunchDimensionsFieldY,
67 RsLaunchDimensionsFieldZ,
68 RsLaunchDimensionsFieldLod,
69 RsLaunchDimensionsFieldFace,
70 RsLaunchDimensionsFieldArray,
71
72 RsLaunchDimensionsFieldCount
73 };
74
75 enum RsExpandKernelDriverInfoPfxField {
76 RsExpandKernelDriverInfoPfxFieldInPtr,
77 RsExpandKernelDriverInfoPfxFieldInStride,
78 RsExpandKernelDriverInfoPfxFieldInLen,
79 RsExpandKernelDriverInfoPfxFieldOutPtr,
80 RsExpandKernelDriverInfoPfxFieldOutStride,
81 RsExpandKernelDriverInfoPfxFieldOutLen,
82 RsExpandKernelDriverInfoPfxFieldDim,
83 RsExpandKernelDriverInfoPfxFieldCurrent,
84 RsExpandKernelDriverInfoPfxFieldUsr,
85 RsExpandKernelDriverInfoPfxFieldUsLenr,
86
87 RsExpandKernelDriverInfoPfxFieldCount
88 };
89
90 llvm::Module *Module;
91 llvm::LLVMContext *Context;
92
93 /*
94 * Pointer to LLVM type information for the the function signature
95 * for expanded kernels. This must be re-calculated for each
96 * module the pass is run on.
97 */
98 llvm::FunctionType *ExpandedFunctionType;
99
100 uint32_t mExportForEachCount;
101 const char **mExportForEachNameList;
102 const uint32_t *mExportForEachSignatureList;
103
104 // Turns on optimization of allocation stride values.
105 bool mEnableStepOpt;
106
getRootSignature(llvm::Function * Function)107 uint32_t getRootSignature(llvm::Function *Function) {
108 const llvm::NamedMDNode *ExportForEachMetadata =
109 Module->getNamedMetadata("#rs_export_foreach");
110
111 if (!ExportForEachMetadata) {
112 llvm::SmallVector<llvm::Type*, 8> RootArgTys;
113 for (llvm::Function::arg_iterator B = Function->arg_begin(),
114 E = Function->arg_end();
115 B != E;
116 ++B) {
117 RootArgTys.push_back(B->getType());
118 }
119
120 // For pre-ICS bitcode, we may not have signature information. In that
121 // case, we use the size of the RootArgTys to select the number of
122 // arguments.
123 return (1 << RootArgTys.size()) - 1;
124 }
125
126 if (ExportForEachMetadata->getNumOperands() == 0) {
127 return 0;
128 }
129
130 bccAssert(ExportForEachMetadata->getNumOperands() > 0);
131
132 // We only handle the case for legacy root() functions here, so this is
133 // hard-coded to look at only the first such function.
134 llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0);
135 if (SigNode != nullptr && SigNode->getNumOperands() == 1) {
136 llvm::Metadata *SigMD = SigNode->getOperand(0);
137 if (llvm::MDString *SigS = llvm::dyn_cast<llvm::MDString>(SigMD)) {
138 llvm::StringRef SigString = SigS->getString();
139 uint32_t Signature = 0;
140 if (SigString.getAsInteger(10, Signature)) {
141 ALOGE("Non-integer signature value '%s'", SigString.str().c_str());
142 return 0;
143 }
144 return Signature;
145 }
146 }
147
148 return 0;
149 }
150
isStepOptSupported(llvm::Type * AllocType)151 bool isStepOptSupported(llvm::Type *AllocType) {
152
153 llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType);
154 llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context);
155
156 if (mEnableStepOpt) {
157 return false;
158 }
159
160 if (AllocType == VoidPtrTy) {
161 return false;
162 }
163
164 if (!PT) {
165 return false;
166 }
167
168 // remaining conditions are 64-bit only
169 if (VoidPtrTy->getPrimitiveSizeInBits() == 32) {
170 return true;
171 }
172
173 // coerce suggests an upconverted struct type, which we can't support
174 if (AllocType->getStructName().find("coerce") != llvm::StringRef::npos) {
175 return false;
176 }
177
178 // 2xi64 and i128 suggest an upconverted struct type, which are also unsupported
179 llvm::Type *V2xi64Ty = llvm::VectorType::get(llvm::Type::getInt64Ty(*Context), 2);
180 llvm::Type *Int128Ty = llvm::Type::getIntNTy(*Context, 128);
181 if (AllocType == V2xi64Ty || AllocType == Int128Ty) {
182 return false;
183 }
184
185 return true;
186 }
187
188 // Get the actual value we should use to step through an allocation.
189 //
190 // Normally the value we use to step through an allocation is given to us by
191 // the driver. However, for certain primitive data types, we can derive an
192 // integer constant for the step value. We use this integer constant whenever
193 // possible to allow further compiler optimizations to take place.
194 //
195 // DL - Target Data size/layout information.
196 // T - Type of allocation (should be a pointer).
197 // OrigStep - Original step increment (root.expand() input from driver).
getStepValue(llvm::DataLayout * DL,llvm::Type * AllocType,llvm::Value * OrigStep)198 llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *AllocType,
199 llvm::Value *OrigStep) {
200 bccAssert(DL);
201 bccAssert(AllocType);
202 bccAssert(OrigStep);
203 llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType);
204 if (isStepOptSupported(AllocType)) {
205 llvm::Type *ET = PT->getElementType();
206 uint64_t ETSize = DL->getTypeAllocSize(ET);
207 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
208 return llvm::ConstantInt::get(Int32Ty, ETSize);
209 } else {
210 return OrigStep;
211 }
212 }
213
214 /// Builds the types required by the pass for the given context.
buildTypes(void)215 void buildTypes(void) {
216 // Create the RsLaunchDimensionsTy and RsExpandKernelDriverInfoPfxTy structs.
217
218 llvm::Type *Int8Ty = llvm::Type::getInt8Ty(*Context);
219 llvm::Type *Int8PtrTy = Int8Ty->getPointerTo();
220 llvm::Type *Int8PtrArrayInputLimitTy = llvm::ArrayType::get(Int8PtrTy, RS_KERNEL_INPUT_LIMIT);
221 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
222 llvm::Type *Int32ArrayInputLimitTy = llvm::ArrayType::get(Int32Ty, RS_KERNEL_INPUT_LIMIT);
223 llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context);
224 llvm::Type *Int32Array4Ty = llvm::ArrayType::get(Int32Ty, 4);
225
226 /* Defined in frameworks/base/libs/rs/cpu_ref/rsCpuCore.h:
227 *
228 * struct RsLaunchDimensions {
229 * uint32_t x;
230 * uint32_t y;
231 * uint32_t z;
232 * uint32_t lod;
233 * uint32_t face;
234 * uint32_t array[4];
235 * };
236 */
237 llvm::SmallVector<llvm::Type*, RsLaunchDimensionsFieldCount> RsLaunchDimensionsTypes;
238 RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t x
239 RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t y
240 RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t z
241 RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t lod
242 RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t face
243 RsLaunchDimensionsTypes.push_back(Int32Array4Ty); // uint32_t array[4]
244 llvm::StructType *RsLaunchDimensionsTy =
245 llvm::StructType::create(RsLaunchDimensionsTypes, "RsLaunchDimensions");
246
247 /* Defined as the beginning of RsExpandKernelDriverInfo in frameworks/base/libs/rs/cpu_ref/rsCpuCoreRuntime.h:
248 *
249 * struct RsExpandKernelDriverInfoPfx {
250 * const uint8_t *inPtr[RS_KERNEL_INPUT_LIMIT];
251 * uint32_t inStride[RS_KERNEL_INPUT_LIMIT];
252 * uint32_t inLen;
253 *
254 * uint8_t *outPtr[RS_KERNEL_INPUT_LIMIT];
255 * uint32_t outStride[RS_KERNEL_INPUT_LIMIT];
256 * uint32_t outLen;
257 *
258 * // Dimension of the launch
259 * RsLaunchDimensions dim;
260 *
261 * // The walking iterator of the launch
262 * RsLaunchDimensions current;
263 *
264 * const void *usr;
265 * uint32_t usrLen;
266 *
267 * // Items below this line are not used by the compiler and can be change in the driver.
268 * // So the compiler must assume there are an unknown number of fields of unknown type
269 * // beginning here.
270 * };
271 *
272 * The name "RsExpandKernelDriverInfoPfx" is known to RSInvariantPass (RSInvariant.cpp).
273 */
274 llvm::SmallVector<llvm::Type*, RsExpandKernelDriverInfoPfxFieldCount> RsExpandKernelDriverInfoPfxTypes;
275 RsExpandKernelDriverInfoPfxTypes.push_back(Int8PtrArrayInputLimitTy); // const uint8_t *inPtr[RS_KERNEL_INPUT_LIMIT]
276 RsExpandKernelDriverInfoPfxTypes.push_back(Int32ArrayInputLimitTy); // uint32_t inStride[RS_KERNEL_INPUT_LIMIT]
277 RsExpandKernelDriverInfoPfxTypes.push_back(Int32Ty); // uint32_t inLen
278 RsExpandKernelDriverInfoPfxTypes.push_back(Int8PtrArrayInputLimitTy); // uint8_t *outPtr[RS_KERNEL_INPUT_LIMIT]
279 RsExpandKernelDriverInfoPfxTypes.push_back(Int32ArrayInputLimitTy); // uint32_t outStride[RS_KERNEL_INPUT_LIMIT]
280 RsExpandKernelDriverInfoPfxTypes.push_back(Int32Ty); // uint32_t outLen
281 RsExpandKernelDriverInfoPfxTypes.push_back(RsLaunchDimensionsTy); // RsLaunchDimensions dim
282 RsExpandKernelDriverInfoPfxTypes.push_back(RsLaunchDimensionsTy); // RsLaunchDimensions current
283 RsExpandKernelDriverInfoPfxTypes.push_back(VoidPtrTy); // const void *usr
284 RsExpandKernelDriverInfoPfxTypes.push_back(Int32Ty); // uint32_t usrLen
285 llvm::StructType *RsExpandKernelDriverInfoPfxTy =
286 llvm::StructType::create(RsExpandKernelDriverInfoPfxTypes, "RsExpandKernelDriverInfoPfx");
287
288 // Create the function type for expanded kernels.
289
290 llvm::Type *RsExpandKernelDriverInfoPfxPtrTy = RsExpandKernelDriverInfoPfxTy->getPointerTo();
291
292 llvm::SmallVector<llvm::Type*, 8> ParamTypes;
293 ParamTypes.push_back(RsExpandKernelDriverInfoPfxPtrTy); // const RsExpandKernelDriverInfoPfx *p
294 ParamTypes.push_back(Int32Ty); // uint32_t x1
295 ParamTypes.push_back(Int32Ty); // uint32_t x2
296 ParamTypes.push_back(Int32Ty); // uint32_t outstep
297
298 ExpandedFunctionType =
299 llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), ParamTypes,
300 false);
301 }
302
303 /// @brief Create skeleton of the expanded function.
304 ///
305 /// This creates a function with the following signature:
306 ///
307 /// void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2,
308 /// uint32_t outstep)
309 ///
createEmptyExpandedFunction(llvm::StringRef OldName)310 llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) {
311 llvm::Function *ExpandedFunction =
312 llvm::Function::Create(ExpandedFunctionType,
313 llvm::GlobalValue::ExternalLinkage,
314 OldName + ".expand", Module);
315
316 bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS);
317
318 llvm::Function::arg_iterator AI = ExpandedFunction->arg_begin();
319
320 (AI++)->setName("p");
321 (AI++)->setName("x1");
322 (AI++)->setName("x2");
323 (AI++)->setName("arg_outstep");
324
325 llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin",
326 ExpandedFunction);
327 llvm::IRBuilder<> Builder(Begin);
328 Builder.CreateRetVoid();
329
330 return ExpandedFunction;
331 }
332
333 /// @brief Create an empty loop
334 ///
335 /// Create a loop of the form:
336 ///
337 /// for (i = LowerBound; i < UpperBound; i++)
338 /// ;
339 ///
340 /// After the loop has been created, the builder is set such that
341 /// instructions can be added to the loop body.
342 ///
343 /// @param Builder The builder to use to build this loop. The current
344 /// position of the builder is the position the loop
345 /// will be inserted.
346 /// @param LowerBound The first value of the loop iterator
347 /// @param UpperBound The maximal value of the loop iterator
348 /// @param LoopIV A reference that will be set to the loop iterator.
349 /// @return The BasicBlock that will be executed after the loop.
createLoop(llvm::IRBuilder<> & Builder,llvm::Value * LowerBound,llvm::Value * UpperBound,llvm::PHINode ** LoopIV)350 llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder,
351 llvm::Value *LowerBound,
352 llvm::Value *UpperBound,
353 llvm::PHINode **LoopIV) {
354 assert(LowerBound->getType() == UpperBound->getType());
355
356 llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB;
357 llvm::Value *Cond, *IVNext;
358 llvm::PHINode *IV;
359
360 CondBB = Builder.GetInsertBlock();
361 AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), nullptr, nullptr);
362 HeaderBB = llvm::BasicBlock::Create(*Context, "Loop", CondBB->getParent());
363
364 // if (LowerBound < Upperbound)
365 // goto LoopHeader
366 // else
367 // goto AfterBB
368 CondBB->getTerminator()->eraseFromParent();
369 Builder.SetInsertPoint(CondBB);
370 Cond = Builder.CreateICmpULT(LowerBound, UpperBound);
371 Builder.CreateCondBr(Cond, HeaderBB, AfterBB);
372
373 // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ]
374 // iv.next = iv + 1
375 // if (iv.next < Upperbound)
376 // goto LoopHeader
377 // else
378 // goto AfterBB
379 Builder.SetInsertPoint(HeaderBB);
380 IV = Builder.CreatePHI(LowerBound->getType(), 2, "X");
381 IV->addIncoming(LowerBound, CondBB);
382 IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1));
383 IV->addIncoming(IVNext, HeaderBB);
384 Cond = Builder.CreateICmpULT(IVNext, UpperBound);
385 Builder.CreateCondBr(Cond, HeaderBB, AfterBB);
386 AfterBB->setName("Exit");
387 Builder.SetInsertPoint(HeaderBB->getFirstNonPHI());
388 *LoopIV = IV;
389 return AfterBB;
390 }
391
392 // Finish building the outgoing argument list for calling a ForEach-able function.
393 //
394 // ArgVector - on input, the non-special arguments
395 // on output, the non-special arguments combined with the special arguments
396 // from SpecialArgVector
397 // SpecialArgVector - special arguments (from ExpandSpecialArguments())
398 // SpecialArgContextIdx - return value of ExpandSpecialArguments()
399 // (position of context argument in SpecialArgVector)
400 // CalleeFunction - the ForEach-able function being called
401 // Builder - for inserting code into the caller function
402 template<unsigned int ArgVectorLen, unsigned int SpecialArgVectorLen>
finishArgList(llvm::SmallVector<llvm::Value *,ArgVectorLen> & ArgVector,const llvm::SmallVector<llvm::Value *,SpecialArgVectorLen> & SpecialArgVector,const int SpecialArgContextIdx,const llvm::Function & CalleeFunction,llvm::IRBuilder<> & CallerBuilder)403 void finishArgList( llvm::SmallVector<llvm::Value *, ArgVectorLen> &ArgVector,
404 const llvm::SmallVector<llvm::Value *, SpecialArgVectorLen> &SpecialArgVector,
405 const int SpecialArgContextIdx,
406 const llvm::Function &CalleeFunction,
407 llvm::IRBuilder<> &CallerBuilder) {
408 /* The context argument (if any) is a pointer to an opaque user-visible type that differs from
409 * the RsExpandKernelDriverInfoPfx type used in the function we are generating (although the
410 * two types represent the same thing). Therefore, we must introduce a pointer cast when
411 * generating a call to the kernel function.
412 */
413 const int ArgContextIdx =
414 SpecialArgContextIdx >= 0 ? (ArgVector.size() + SpecialArgContextIdx) : SpecialArgContextIdx;
415 ArgVector.append(SpecialArgVector.begin(), SpecialArgVector.end());
416 if (ArgContextIdx >= 0) {
417 llvm::Type *ContextArgType = nullptr;
418 int ArgIdx = ArgContextIdx;
419 for (const auto &Arg : CalleeFunction.getArgumentList()) {
420 if (!ArgIdx--) {
421 ContextArgType = Arg.getType();
422 break;
423 }
424 }
425 bccAssert(ContextArgType);
426 ArgVector[ArgContextIdx] = CallerBuilder.CreatePointerCast(ArgVector[ArgContextIdx], ContextArgType);
427 }
428 }
429
430 public:
RSForEachExpandPass(bool pEnableStepOpt=true)431 RSForEachExpandPass(bool pEnableStepOpt = true)
432 : ModulePass(ID), Module(nullptr), Context(nullptr),
433 mEnableStepOpt(pEnableStepOpt) {
434
435 }
436
getAnalysisUsage(llvm::AnalysisUsage & AU) const437 virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
438 // This pass does not use any other analysis passes, but it does
439 // add/wrap the existing functions in the module (thus altering the CFG).
440 }
441
442 // Build contribution to outgoing argument list for calling a
443 // ForEach-able function, based on the special parameters of that
444 // function.
445 //
446 // Signature - metadata bits for the signature of the ForEach-able function
447 // X, Arg_p - values derived directly from expanded function,
448 // suitable for computing arguments for the ForEach-able function
449 // CalleeArgs - contribution is accumulated here
450 // Bump - invoked once for each contributed outgoing argument
451 //
452 // Return value is the (zero-based) position of the context (Arg_p)
453 // argument in the CalleeArgs vector, or a negative value if the
454 // context argument is not placed in the CalleeArgs vector.
ExpandSpecialArguments(uint32_t Signature,llvm::Value * X,llvm::Value * Arg_p,llvm::IRBuilder<> & Builder,llvm::SmallVector<llvm::Value *,8> & CalleeArgs,std::function<void ()> Bump)455 int ExpandSpecialArguments(uint32_t Signature,
456 llvm::Value *X,
457 llvm::Value *Arg_p,
458 llvm::IRBuilder<> &Builder,
459 llvm::SmallVector<llvm::Value*, 8> &CalleeArgs,
460 std::function<void ()> Bump) {
461
462 bccAssert(CalleeArgs.empty());
463
464 int Return = -1;
465 if (bcinfo::MetadataExtractor::hasForEachSignatureCtxt(Signature)) {
466 CalleeArgs.push_back(Arg_p);
467 Bump();
468 Return = CalleeArgs.size() - 1;
469 }
470
471 if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) {
472 CalleeArgs.push_back(X);
473 Bump();
474 }
475
476 if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature) ||
477 bcinfo::MetadataExtractor::hasForEachSignatureZ(Signature)) {
478
479 llvm::Value *Current = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldCurrent);
480
481 if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) {
482 llvm::Value *Y = Builder.CreateLoad(
483 Builder.CreateStructGEP(nullptr, Current, RsLaunchDimensionsFieldY), "Y");
484
485 CalleeArgs.push_back(Y);
486 Bump();
487 }
488
489 if (bcinfo::MetadataExtractor::hasForEachSignatureZ(Signature)) {
490 llvm::Value *Z = Builder.CreateLoad(
491 Builder.CreateStructGEP(nullptr, Current, RsLaunchDimensionsFieldZ), "Z");
492 CalleeArgs.push_back(Z);
493 Bump();
494 }
495 }
496
497 return Return;
498 }
499
500 /* Performs the actual optimization on a selected function. On success, the
501 * Module will contain a new function of the name "<NAME>.expand" that
502 * invokes <NAME>() in a loop with the appropriate parameters.
503 */
ExpandFunction(llvm::Function * Function,uint32_t Signature)504 bool ExpandFunction(llvm::Function *Function, uint32_t Signature) {
505 ALOGV("Expanding ForEach-able Function %s",
506 Function->getName().str().c_str());
507
508 if (!Signature) {
509 Signature = getRootSignature(Function);
510 if (!Signature) {
511 // We couldn't determine how to expand this function based on its
512 // function signature.
513 return false;
514 }
515 }
516
517 llvm::DataLayout DL(Module);
518
519 llvm::Function *ExpandedFunction =
520 createEmptyExpandedFunction(Function->getName());
521
522 /*
523 * Extract the expanded function's parameters. It is guaranteed by
524 * createEmptyExpandedFunction that there will be five parameters.
525 */
526
527 bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS);
528
529 llvm::Function::arg_iterator ExpandedFunctionArgIter =
530 ExpandedFunction->arg_begin();
531
532 llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++);
533 llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++);
534 llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++);
535 llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter);
536
537 llvm::Value *InStep = nullptr;
538 llvm::Value *OutStep = nullptr;
539
540 // Construct the actual function body.
541 llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin());
542
543 // Collect and construct the arguments for the kernel().
544 // Note that we load any loop-invariant arguments before entering the Loop.
545 llvm::Function::arg_iterator FunctionArgIter = Function->arg_begin();
546
547 llvm::Type *InTy = nullptr;
548 llvm::Value *InBasePtr = nullptr;
549 if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) {
550 llvm::Value *InsBasePtr = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInPtr, "inputs_base");
551
552 llvm::Value *InStepsBase = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInStride, "insteps_base");
553
554 llvm::Value *InStepAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InStepsBase, 0, 0);
555 llvm::LoadInst *InStepArg = Builder.CreateLoad(InStepAddr,
556 "instep_addr");
557
558 InTy = (FunctionArgIter++)->getType();
559 InStep = getStepValue(&DL, InTy, InStepArg);
560
561 InStep->setName("instep");
562
563 llvm::Value *InputAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InsBasePtr, 0, 0);
564 InBasePtr = Builder.CreateLoad(InputAddr, "input_base");
565 }
566
567 llvm::Type *OutTy = nullptr;
568 llvm::Value *OutBasePtr = nullptr;
569 if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) {
570 OutTy = (FunctionArgIter++)->getType();
571 OutStep = getStepValue(&DL, OutTy, Arg_outstep);
572 OutStep->setName("outstep");
573 OutBasePtr = Builder.CreateLoad(
574 Builder.CreateConstInBoundsGEP2_32(nullptr,
575 Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldOutPtr),
576 0, 0));
577 }
578
579 llvm::Value *UsrData = nullptr;
580 if (bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)) {
581 llvm::Type *UsrDataTy = (FunctionArgIter++)->getType();
582 UsrData = Builder.CreatePointerCast(Builder.CreateLoad(
583 Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldUsr)), UsrDataTy);
584 UsrData->setName("UsrData");
585 }
586
587 llvm::PHINode *IV;
588 createLoop(Builder, Arg_x1, Arg_x2, &IV);
589
590 llvm::SmallVector<llvm::Value*, 8> CalleeArgs;
591 const int CalleeArgsContextIdx = ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs,
592 [&FunctionArgIter]() { FunctionArgIter++; });
593
594 bccAssert(FunctionArgIter == Function->arg_end());
595
596 // Populate the actual call to kernel().
597 llvm::SmallVector<llvm::Value*, 8> RootArgs;
598
599 llvm::Value *InPtr = nullptr;
600 llvm::Value *OutPtr = nullptr;
601
602 // Calculate the current input and output pointers
603 //
604 // We always calculate the input/output pointers with a GEP operating on i8
605 // values and only cast at the very end to OutTy. This is because the step
606 // between two values is given in bytes.
607 //
608 // TODO: We could further optimize the output by using a GEP operation of
609 // type 'OutTy' in cases where the element type of the allocation allows.
610 if (OutBasePtr) {
611 llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1);
612 OutOffset = Builder.CreateMul(OutOffset, OutStep);
613 OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset);
614 OutPtr = Builder.CreatePointerCast(OutPtr, OutTy);
615 }
616
617 if (InBasePtr) {
618 llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1);
619 InOffset = Builder.CreateMul(InOffset, InStep);
620 InPtr = Builder.CreateGEP(InBasePtr, InOffset);
621 InPtr = Builder.CreatePointerCast(InPtr, InTy);
622 }
623
624 if (InPtr) {
625 RootArgs.push_back(InPtr);
626 }
627
628 if (OutPtr) {
629 RootArgs.push_back(OutPtr);
630 }
631
632 if (UsrData) {
633 RootArgs.push_back(UsrData);
634 }
635
636 finishArgList(RootArgs, CalleeArgs, CalleeArgsContextIdx, *Function, Builder);
637
638 Builder.CreateCall(Function, RootArgs);
639
640 return true;
641 }
642
643 /* Expand a pass-by-value kernel.
644 */
ExpandKernel(llvm::Function * Function,uint32_t Signature)645 bool ExpandKernel(llvm::Function *Function, uint32_t Signature) {
646 bccAssert(bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature));
647 ALOGV("Expanding kernel Function %s", Function->getName().str().c_str());
648
649 // TODO: Refactor this to share functionality with ExpandFunction.
650 llvm::DataLayout DL(Module);
651
652 llvm::Function *ExpandedFunction =
653 createEmptyExpandedFunction(Function->getName());
654
655 /*
656 * Extract the expanded function's parameters. It is guaranteed by
657 * createEmptyExpandedFunction that there will be five parameters.
658 */
659
660 bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS);
661
662 llvm::Function::arg_iterator ExpandedFunctionArgIter =
663 ExpandedFunction->arg_begin();
664
665 llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++);
666 llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++);
667 llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++);
668 llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter);
669
670 // Construct the actual function body.
671 llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin());
672
673 // Create TBAA meta-data.
674 llvm::MDNode *TBAARenderScriptDistinct, *TBAARenderScript,
675 *TBAAAllocation, *TBAAPointer;
676 llvm::MDBuilder MDHelper(*Context);
677
678 TBAARenderScriptDistinct =
679 MDHelper.createTBAARoot("RenderScript Distinct TBAA");
680 TBAARenderScript = MDHelper.createTBAANode("RenderScript TBAA",
681 TBAARenderScriptDistinct);
682 TBAAAllocation = MDHelper.createTBAAScalarTypeNode("allocation",
683 TBAARenderScript);
684 TBAAAllocation = MDHelper.createTBAAStructTagNode(TBAAAllocation,
685 TBAAAllocation, 0);
686 TBAAPointer = MDHelper.createTBAAScalarTypeNode("pointer",
687 TBAARenderScript);
688 TBAAPointer = MDHelper.createTBAAStructTagNode(TBAAPointer, TBAAPointer, 0);
689
690 llvm::MDNode *AliasingDomain, *AliasingScope;
691 AliasingDomain = MDHelper.createAnonymousAliasScopeDomain("RS argument scope domain");
692 AliasingScope = MDHelper.createAnonymousAliasScope(AliasingDomain, "RS argument scope");
693
694 /*
695 * Collect and construct the arguments for the kernel().
696 *
697 * Note that we load any loop-invariant arguments before entering the Loop.
698 */
699 size_t NumInputs = Function->arg_size();
700
701 // No usrData parameter on kernels.
702 bccAssert(
703 !bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature));
704
705 llvm::Function::arg_iterator ArgIter = Function->arg_begin();
706
707 // Check the return type
708 llvm::Type *OutTy = nullptr;
709 llvm::Value *OutStep = nullptr;
710 llvm::LoadInst *OutBasePtr = nullptr;
711 llvm::Value *CastedOutBasePtr = nullptr;
712
713 bool PassOutByPointer = false;
714
715 if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) {
716 llvm::Type *OutBaseTy = Function->getReturnType();
717
718 if (OutBaseTy->isVoidTy()) {
719 PassOutByPointer = true;
720 OutTy = ArgIter->getType();
721
722 ArgIter++;
723 --NumInputs;
724 } else {
725 // We don't increment Args, since we are using the actual return type.
726 OutTy = OutBaseTy->getPointerTo();
727 }
728
729 OutStep = getStepValue(&DL, OutTy, Arg_outstep);
730 OutStep->setName("outstep");
731 OutBasePtr = Builder.CreateLoad(
732 Builder.CreateConstInBoundsGEP2_32(nullptr,
733 Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldOutPtr),
734 0, 0));
735
736 if (gEnableRsTbaa) {
737 OutBasePtr->setMetadata("tbaa", TBAAPointer);
738 }
739
740 OutBasePtr->setMetadata("alias.scope", AliasingScope);
741
742 CastedOutBasePtr = Builder.CreatePointerCast(OutBasePtr, OutTy, "casted_out");
743 }
744
745 llvm::PHINode *IV;
746 createLoop(Builder, Arg_x1, Arg_x2, &IV);
747
748 llvm::SmallVector<llvm::Value*, 8> CalleeArgs;
749 const int CalleeArgsContextIdx = ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs,
750 [&NumInputs]() { --NumInputs; });
751
752 llvm::SmallVector<llvm::Type*, 8> InTypes;
753 llvm::SmallVector<llvm::Value*, 8> InSteps;
754 llvm::SmallVector<llvm::Value*, 8> InBasePtrs;
755 llvm::SmallVector<llvm::Value*, 8> InStructTempSlots;
756
757 bccAssert(NumInputs <= RS_KERNEL_INPUT_LIMIT);
758
759 if (NumInputs > 0) {
760 llvm::Value *InsBasePtr = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInPtr, "inputs_base");
761
762 llvm::Value *InStepsBase = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInStride, "insteps_base");
763
764 llvm::Instruction *AllocaInsertionPoint = &*ExpandedFunction->getEntryBlock().begin();
765 for (size_t InputIndex = 0; InputIndex < NumInputs;
766 ++InputIndex, ArgIter++) {
767
768 llvm::Value *InStepAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InStepsBase, 0, InputIndex);
769 llvm::LoadInst *InStepArg = Builder.CreateLoad(InStepAddr,
770 "instep_addr");
771
772 llvm::Type *InType = ArgIter->getType();
773
774 /*
775 * AArch64 calling conventions dictate that structs of sufficient size
776 * get passed by pointer instead of passed by value. This, combined
777 * with the fact that we don't allow kernels to operate on pointer
778 * data means that if we see a kernel with a pointer parameter we know
779 * that it is struct input that has been promoted. As such we don't
780 * need to convert its type to a pointer. Later we will need to know
781 * to create a temporary copy on the stack, so we save this information
782 * in InStructTempSlots.
783 */
784 if (auto PtrType = llvm::dyn_cast<llvm::PointerType>(InType)) {
785 llvm::Type *ElementType = PtrType->getElementType();
786 uint64_t Alignment = DL.getABITypeAlignment(ElementType);
787 llvm::Value *Slot = new llvm::AllocaInst(ElementType,
788 nullptr,
789 Alignment,
790 "input_struct_slot",
791 AllocaInsertionPoint);
792 InStructTempSlots.push_back(Slot);
793 } else {
794 InType = InType->getPointerTo();
795 InStructTempSlots.push_back(nullptr);
796 }
797
798 llvm::Value *InStep = getStepValue(&DL, InType, InStepArg);
799
800 InStep->setName("instep");
801
802 llvm::Value *InputAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InsBasePtr, 0, InputIndex);
803 llvm::LoadInst *InBasePtr = Builder.CreateLoad(InputAddr,
804 "input_base");
805 llvm::Value *CastInBasePtr = Builder.CreatePointerCast(InBasePtr,
806 InType, "casted_in");
807 if (gEnableRsTbaa) {
808 InBasePtr->setMetadata("tbaa", TBAAPointer);
809 }
810
811 InBasePtr->setMetadata("alias.scope", AliasingScope);
812
813 InTypes.push_back(InType);
814 InSteps.push_back(InStep);
815 InBasePtrs.push_back(CastInBasePtr);
816 }
817 }
818
819 // Populate the actual call to kernel().
820 llvm::SmallVector<llvm::Value*, 8> RootArgs;
821
822 // Calculate the current input and output pointers
823 //
824 //
825 // We always calculate the input/output pointers with a GEP operating on i8
826 // values combined with a multiplication and only cast at the very end to
827 // OutTy. This is to account for dynamic stepping sizes when the value
828 // isn't apparent at compile time. In the (very common) case when we know
829 // the step size at compile time, due to haveing complete type information
830 // this multiplication will optmized out and produces code equivalent to a
831 // a GEP on a pointer of the correct type.
832
833 // Output
834
835 llvm::Value *OutPtr = nullptr;
836 if (CastedOutBasePtr) {
837 llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1);
838
839 OutPtr = Builder.CreateGEP(CastedOutBasePtr, OutOffset);
840
841 if (PassOutByPointer) {
842 RootArgs.push_back(OutPtr);
843 }
844 }
845
846 // Inputs
847
848 if (NumInputs > 0) {
849 llvm::Value *Offset = Builder.CreateSub(IV, Arg_x1);
850
851 for (size_t Index = 0; Index < NumInputs; ++Index) {
852 llvm::Value *InPtr = Builder.CreateGEP(InBasePtrs[Index], Offset);
853 llvm::Value *Input;
854
855 if (llvm::Value *TemporarySlot = InStructTempSlots[Index]) {
856 // Pass a pointer to a temporary on the stack, rather than
857 // passing a pointer to the original value. We do not want
858 // the kernel to potentially modify the input data.
859
860 llvm::Type *ElementType = llvm::cast<llvm::PointerType>(
861 InPtr->getType())->getElementType();
862 uint64_t StoreSize = DL.getTypeStoreSize(ElementType);
863 uint64_t Alignment = DL.getABITypeAlignment(ElementType);
864
865 Builder.CreateMemCpy(TemporarySlot, InPtr, StoreSize, Alignment,
866 /* isVolatile = */ false,
867 /* !tbaa = */ gEnableRsTbaa ? TBAAAllocation : nullptr,
868 /* !tbaa.struct = */ nullptr,
869 /* !alias.scope = */ AliasingScope);
870 Input = TemporarySlot;
871 } else {
872 llvm::LoadInst *InputLoad = Builder.CreateLoad(InPtr, "input");
873
874 if (gEnableRsTbaa) {
875 InputLoad->setMetadata("tbaa", TBAAAllocation);
876 }
877
878 InputLoad->setMetadata("alias.scope", AliasingScope);
879
880 Input = InputLoad;
881 }
882
883 RootArgs.push_back(Input);
884 }
885 }
886
887 finishArgList(RootArgs, CalleeArgs, CalleeArgsContextIdx, *Function, Builder);
888
889 llvm::Value *RetVal = Builder.CreateCall(Function, RootArgs);
890
891 if (OutPtr && !PassOutByPointer) {
892 llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr);
893 if (gEnableRsTbaa) {
894 Store->setMetadata("tbaa", TBAAAllocation);
895 }
896 Store->setMetadata("alias.scope", AliasingScope);
897 }
898
899 return true;
900 }
901
902 /// @brief Checks if pointers to allocation internals are exposed
903 ///
904 /// This function verifies if through the parameters passed to the kernel
905 /// or through calls to the runtime library the script gains access to
906 /// pointers pointing to data within a RenderScript Allocation.
907 /// If we know we control all loads from and stores to data within
908 /// RenderScript allocations and if we know the run-time internal accesses
909 /// are all annotated with RenderScript TBAA metadata, only then we
910 /// can safely use TBAA to distinguish between generic and from-allocation
911 /// pointers.
allocPointersExposed(llvm::Module & Module)912 bool allocPointersExposed(llvm::Module &Module) {
913 // Old style kernel function can expose pointers to elements within
914 // allocations.
915 // TODO: Extend analysis to allow simple cases of old-style kernels.
916 for (size_t i = 0; i < mExportForEachCount; ++i) {
917 const char *Name = mExportForEachNameList[i];
918 uint32_t Signature = mExportForEachSignatureList[i];
919 if (Module.getFunction(Name) &&
920 !bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)) {
921 return true;
922 }
923 }
924
925 // Check for library functions that expose a pointer to an Allocation or
926 // that are not yet annotated with RenderScript-specific tbaa information.
927 static std::vector<std::string> Funcs;
928
929 // rsGetElementAt(...)
930 Funcs.push_back("_Z14rsGetElementAt13rs_allocationj");
931 Funcs.push_back("_Z14rsGetElementAt13rs_allocationjj");
932 Funcs.push_back("_Z14rsGetElementAt13rs_allocationjjj");
933 // rsSetElementAt()
934 Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvj");
935 Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjj");
936 Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjjj");
937 // rsGetElementAtYuv_uchar_Y()
938 Funcs.push_back("_Z25rsGetElementAtYuv_uchar_Y13rs_allocationjj");
939 // rsGetElementAtYuv_uchar_U()
940 Funcs.push_back("_Z25rsGetElementAtYuv_uchar_U13rs_allocationjj");
941 // rsGetElementAtYuv_uchar_V()
942 Funcs.push_back("_Z25rsGetElementAtYuv_uchar_V13rs_allocationjj");
943
944 for (std::vector<std::string>::iterator FI = Funcs.begin(),
945 FE = Funcs.end();
946 FI != FE; ++FI) {
947 llvm::Function *Function = Module.getFunction(*FI);
948
949 if (!Function) {
950 ALOGE("Missing run-time function '%s'", FI->c_str());
951 return true;
952 }
953
954 if (Function->getNumUses() > 0) {
955 return true;
956 }
957 }
958
959 return false;
960 }
961
962 /// @brief Connect RenderScript TBAA metadata to C/C++ metadata
963 ///
964 /// The TBAA metadata used to annotate loads/stores from RenderScript
965 /// Allocations is generated in a separate TBAA tree with a
966 /// "RenderScript Distinct TBAA" root node. LLVM does assume may-alias for
967 /// all nodes in unrelated alias analysis trees. This function makes the
968 /// "RenderScript TBAA" node (which is parented by the Distinct TBAA root),
969 /// a subtree of the normal C/C++ TBAA tree aside of normal C/C++ types. With
970 /// the connected trees every access to an Allocation is resolved to
971 /// must-alias if compared to a normal C/C++ access.
connectRenderScriptTBAAMetadata(llvm::Module & Module)972 void connectRenderScriptTBAAMetadata(llvm::Module &Module) {
973 llvm::MDBuilder MDHelper(*Context);
974 llvm::MDNode *TBAARenderScriptDistinct =
975 MDHelper.createTBAARoot("RenderScript Distinct TBAA");
976 llvm::MDNode *TBAARenderScript = MDHelper.createTBAANode(
977 "RenderScript TBAA", TBAARenderScriptDistinct);
978 llvm::MDNode *TBAARoot = MDHelper.createTBAARoot("Simple C/C++ TBAA");
979 TBAARenderScript->replaceOperandWith(1, TBAARoot);
980 }
981
runOnModule(llvm::Module & Module)982 virtual bool runOnModule(llvm::Module &Module) {
983 bool Changed = false;
984 this->Module = &Module;
985 this->Context = &Module.getContext();
986
987 this->buildTypes();
988
989 bcinfo::MetadataExtractor me(&Module);
990 if (!me.extract()) {
991 ALOGE("Could not extract metadata from module!");
992 return false;
993 }
994 mExportForEachCount = me.getExportForEachSignatureCount();
995 mExportForEachNameList = me.getExportForEachNameList();
996 mExportForEachSignatureList = me.getExportForEachSignatureList();
997
998 bool AllocsExposed = allocPointersExposed(Module);
999
1000 for (size_t i = 0; i < mExportForEachCount; ++i) {
1001 const char *name = mExportForEachNameList[i];
1002 uint32_t signature = mExportForEachSignatureList[i];
1003 llvm::Function *kernel = Module.getFunction(name);
1004 if (kernel) {
1005 if (bcinfo::MetadataExtractor::hasForEachSignatureKernel(signature)) {
1006 Changed |= ExpandKernel(kernel, signature);
1007 kernel->setLinkage(llvm::GlobalValue::InternalLinkage);
1008 } else if (kernel->getReturnType()->isVoidTy()) {
1009 Changed |= ExpandFunction(kernel, signature);
1010 kernel->setLinkage(llvm::GlobalValue::InternalLinkage);
1011 } else {
1012 // There are some graphics root functions that are not
1013 // expanded, but that will be called directly. For those
1014 // functions, we can not set the linkage to internal.
1015 }
1016 }
1017 }
1018
1019 if (gEnableRsTbaa && !AllocsExposed) {
1020 connectRenderScriptTBAAMetadata(Module);
1021 }
1022
1023 return Changed;
1024 }
1025
getPassName() const1026 virtual const char *getPassName() const {
1027 return "ForEach-able Function Expansion";
1028 }
1029
1030 }; // end RSForEachExpandPass
1031
1032 } // end anonymous namespace
1033
1034 char RSForEachExpandPass::ID = 0;
1035 static llvm::RegisterPass<RSForEachExpandPass> X("foreachexp", "ForEach Expand Pass");
1036
1037 namespace bcc {
1038
1039 llvm::ModulePass *
createRSForEachExpandPass(bool pEnableStepOpt)1040 createRSForEachExpandPass(bool pEnableStepOpt){
1041 return new RSForEachExpandPass(pEnableStepOpt);
1042 }
1043
1044 } // end namespace bcc
1045