Lines Matching refs:MatMul

1091                                CallInst *MatMul) {  in getNonAliasingPointer()  argument
1105 BasicBlock *Check0 = MatMul->getParent(); in getNonAliasingPointer()
1113 BasicBlock *Check1 = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, in getNonAliasingPointer()
1116 SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, nullptr, "copy"); in getNonAliasingPointer()
1117 BasicBlock *Fusion = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, in getNonAliasingPointer()
1123 IRBuilder<> Builder(MatMul); in getNonAliasingPointer()
1170 bool isFusionProfitable(CallInst *MatMul) { in isFusionProfitable() argument
1174 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in isFusionProfitable()
1175 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in isFusionProfitable()
1180 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in isFusionProfitable()
1212 void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape, in createTiledLoops() argument
1215 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in createTiledLoops()
1220 Instruction *InsertI = cast<Instruction>(MatMul); in createTiledLoops()
1224 IRBuilder<> Builder(MatMul); in createTiledLoops()
1228 FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize); in createTiledLoops()
1269 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1, in emitSIMDTiling() argument
1274 if (!isFusionProfitable(MatMul)) in emitSIMDTiling()
1277 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in emitSIMDTiling()
1278 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in emitSIMDTiling()
1283 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in emitSIMDTiling()
1285 Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul); in emitSIMDTiling()
1286 Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul); in emitSIMDTiling()
1289 bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) && in emitSIMDTiling()
1290 MatMul->hasAllowContract()); in emitSIMDTiling()
1292 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store, in emitSIMDTiling()
1322 FusedInsts.insert(MatMul); in emitSIMDTiling()
1324 MatMul->eraseFromParent(); in emitSIMDTiling()
1342 void LowerMatrixMultiplyFused(CallInst *MatMul, in LowerMatrixMultiplyFused() argument
1344 if (!FuseMatrix || !MatMul->hasOneUse() || in LowerMatrixMultiplyFused()
1350 auto *LoadOp0 = dyn_cast<LoadInst>(MatMul->getOperand(0)); in LowerMatrixMultiplyFused()
1351 auto *LoadOp1 = dyn_cast<LoadInst>(MatMul->getOperand(1)); in LowerMatrixMultiplyFused()
1352 auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin()); in LowerMatrixMultiplyFused()
1358 if (AddrI && (!DT->dominates(AddrI, MatMul))) in LowerMatrixMultiplyFused()
1361 emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts); in LowerMatrixMultiplyFused()
1367 void LowerMultiply(CallInst *MatMul) { in LowerMultiply() argument
1368 IRBuilder<> Builder(MatMul); in LowerMultiply()
1369 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); in LowerMultiply()
1370 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); in LowerMultiply()
1371 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); in LowerMultiply()
1373 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder); in LowerMultiply()
1374 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder); in LowerMultiply()
1387 bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) && in LowerMultiply()
1388 MatMul->hasAllowContract()); in LowerMultiply()
1390 finalizeLowering(MatMul, Result, Builder); in LowerMultiply()