1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "Conv2DOperationConverter.h"
18 
19 #include <vector>
20 
21 #include "OperationConverterResolver.h"
22 #include "SubGraphContext.h"
23 
24 namespace android {
25 namespace nn {
26 
getConv2DInputs(const Operation & operation,SubGraphContext * context) const27 Result<std::vector<int32_t>> Conv2DOperationConverter::getConv2DInputs(
28         const Operation& operation, SubGraphContext* context) const {
29     NN_RET_CHECK(isOperandConstant(
30             context->getSubgraph()->operands[operation.inputs[kFilterTensorIdx]]));
31 
32     NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kInputTensorIdx]));
33     // TFLite does not support asymmetric tensors for convolution filters
34     NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kFilterTensorIdx],
35                                                       true /* makeSymmetric */));
36     NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kBiasTensorIdx]));
37     std::vector<int32_t> inputs{
38             context->getTensorIdxFromOperandIdx(operation.inputs[kInputTensorIdx]),
39             context->getTensorIdxFromOperandIdx(operation.inputs[kFilterTensorIdx]),
40             context->getTensorIdxFromOperandIdx(operation.inputs[kBiasTensorIdx])};
41     return inputs;
42 }
43 
getConv2DOutputs(const Operation & operation,SubGraphContext * context) const44 Result<std::vector<int32_t>> Conv2DOperationConverter::getConv2DOutputs(
45         const Operation& operation, SubGraphContext* context) const {
46     NN_TRY(context->createTensorFlatbufferFromOperand(operation.outputs[kOutputTensorIdx]));
47     std::vector<int32_t> outputs{
48             context->getTensorIdxFromOperandIdx(operation.outputs[kOutputTensorIdx])};
49     return outputs;
50 }
51 
decomposeExplicitPadding(const Operation & operation,SubGraphContext * context) const52 Result<int> Conv2DOperationConverter::decomposeExplicitPadding(const Operation& operation,
53                                                                SubGraphContext* context) const {
54     const Model::Subgraph* subgraph = context->getSubgraph();
55     const Operand& inputOperand = subgraph->operands[operation.inputs[0]];
56 
57     // add opcode for PAD if it does not exist yet
58     uint32_t opCodeIdx = context->addOpCode(OperationType::PAD);
59 
60     // pad options
61     auto padOptionsFlatbuffer = tflite::CreatePadOptions(context->getBuilder());
62 
63     // check to make sure padding Operands are constants
64     const Operand& frontWidthPaddingOperand = subgraph->operands[operation.inputs[3]];
65     const Operand& backWidthPaddingOperand = subgraph->operands[operation.inputs[4]];
66     const Operand& frontHeightPaddingOperand = subgraph->operands[operation.inputs[5]];
67     const Operand& backHeightPaddingOperand = subgraph->operands[operation.inputs[6]];
68     NN_RET_CHECK(isOperandConstant(frontWidthPaddingOperand));
69     NN_RET_CHECK(isOperandConstant(backWidthPaddingOperand));
70     NN_RET_CHECK(isOperandConstant(frontHeightPaddingOperand));
71     NN_RET_CHECK(isOperandConstant(backHeightPaddingOperand));
72 
73     // get padding params
74     int32_t frontHeightPadding = context->getConstantScalar<int32_t>(frontHeightPaddingOperand);
75     int32_t backHeightPadding = context->getConstantScalar<int32_t>(backHeightPaddingOperand);
76     int32_t frontWidthPadding = context->getConstantScalar<int32_t>(frontWidthPaddingOperand);
77     int32_t backWidthPadding = context->getConstantScalar<int32_t>(backWidthPaddingOperand);
78 
79     // build padding buffer
80     const Dimensions& dims = inputOperand.dimensions;
81     int numDimensionsInput = static_cast<int>(dims.size());
82     std::vector<int32_t> paddingData(numDimensionsInput * 2, 0);
83     paddingData[2] = frontHeightPadding;
84     paddingData[3] = backHeightPadding;
85     paddingData[4] = frontWidthPadding;
86     paddingData[5] = backWidthPadding;
87     uint32_t paddingBufferIdx = context->addBufferFromData(
88             reinterpret_cast<uint8_t*>(paddingData.data()), paddingData.size() * sizeof(int32_t));
89 
90     // create new tensor for padding
91     std::vector<int32_t> padShape{numDimensionsInput, 2};
92     auto padTensor = tflite::CreateTensorDirect(context->getBuilder(), &padShape /* shape */,
93                                                 tflite::TensorType::TensorType_INT32 /* type */,
94                                                 paddingBufferIdx /* buffer */);
95     int padTensorIdx = context->addTensorFlatbuffer(padTensor);
96 
97     // add inputs for padding operation
98     std::vector<int32_t> padInputs = {context->getTensorIdxFromOperandIdx(operation.inputs[0]),
99                                       padTensorIdx};
100 
101     // get dimensions of output of pad operation
102     std::vector<int32_t> padToConv2dShape(dims.begin(), dims.end());
103     // keep unknown height and width dimensions unknown
104     padToConv2dShape[1] = padToConv2dShape[1] != 0
105                                   ? frontHeightPadding + padToConv2dShape[1] + backHeightPadding
106                                   : -1;
107     padToConv2dShape[2] = padToConv2dShape[2] != 0
108                                   ? frontWidthPadding + padToConv2dShape[2] + backWidthPadding
109                                   : -1;
110     replaceZeroDimensions(&padToConv2dShape);
111 
112     // build quantization parameters
113     std::vector<float> scaleVector{inputOperand.scale};
114     std::vector<int64_t> zeroPointVector{inputOperand.zeroPoint};
115     // min and max used to convert TFLite models to TF models, so it is unused in this case and can
116     // be set to 0
117     std::vector<float> minVector{0};
118     std::vector<float> maxVector{0};
119     auto quantizationParams = tflite::CreateQuantizationParametersDirect(
120             context->getBuilder(), &minVector /* min */, &maxVector /* max */,
121             &scaleVector /* scale */, &zeroPointVector /* zero_point */,
122             tflite::QuantizationDetails::QuantizationDetails_NONE /* details_type */);
123 
124     // create new tensor to be output of pad & input for conv2d
125     auto padToConv2dTensor = tflite::CreateTensorDirect(
126             context->getBuilder(), &padToConv2dShape /* shape */,
127             NN_TRY(getTensorFlatbufferOperandType(inputOperand.type)) /* type */, 0 /* buffer */,
128             0 /* name */, quantizationParams /* quantization */);
129     int padToConv2dTensorIdx = context->addTensorFlatbuffer(padToConv2dTensor);
130 
131     // set output for padding operation and add to operators
132     std::vector<int32_t> padOutputs{padToConv2dTensorIdx};
133 
134     OperatorFlatbuffer padOp = tflite::CreateOperatorDirect(
135             context->getBuilder(), opCodeIdx, &padInputs, &padOutputs,
136             tflite::BuiltinOptions::BuiltinOptions_PadOptions, padOptionsFlatbuffer.Union());
137     context->addOperatorFlatbuffer(padOp);
138 
139     // Return tensor index of pad output created
140     return padToConv2dTensorIdx;
141 }
142 
convert(const Operation & operation,SubGraphContext * context) const143 Result<void> Conv2DOperationConverter::convert(const Operation& operation,
144                                                SubGraphContext* context) const {
145     const Model::Subgraph* subgraph = context->getSubgraph();
146 
147     // add opcode for CONV_2D if not added yet
148     uint32_t opCodeIdx = context->addOpCode(OperationType::CONV_2D);
149 
150     // if there are less than 8 inputs or the input at the 7th index is a BOOL, there is implicit
151     // padding
152     bool isImplicitPadding = false;
153     if (operation.inputs.size() < 8 ||
154         subgraph->operands[operation.inputs[7]].type == OperandType::BOOL) {
155         isImplicitPadding = true;
156     }
157 
158     std::vector<int32_t> inputs = NN_TRY(getConv2DInputs(operation, context));
159     std::vector<int32_t> outputs = NN_TRY(getConv2DOutputs(operation, context));
160 
161     // if explicit padding, we need to decompose the operation to a separate padding op and a conv2d
162     // op
163     if (!isImplicitPadding) {
164         auto padOpIdx = NN_TRY(decomposeExplicitPadding(operation, context));
165         inputs[0] = padOpIdx;
166     }
167 
168     int baseOptionsIdx = 4;
169     tflite::Padding padding;
170     if (isImplicitPadding) {
171         const Operand& paddingTypeOperand = subgraph->operands[operation.inputs[3]];
172         NN_RET_CHECK(isOperandConstant(paddingTypeOperand));
173 
174         int32_t paddingType = context->getConstantScalar<int32_t>(paddingTypeOperand);
175         padding = getTFLitePadding(paddingType);
176     } else {
177         padding = tflite::Padding::Padding_VALID;
178         baseOptionsIdx = 7;
179     }
180 
181     // check if stride and activation Operands are constant
182     const Operand& strideWOperand =
183             subgraph->operands[operation.inputs[baseOptionsIdx + kStrideWOffset]];
184     const Operand& strideHOperand =
185             subgraph->operands[operation.inputs[baseOptionsIdx + kStrideHOffset]];
186     const Operand& activationOperand =
187             subgraph->operands[operation.inputs[baseOptionsIdx + kActivationOffset]];
188     NN_RET_CHECK(isOperandConstant(strideWOperand));
189     NN_RET_CHECK(isOperandConstant(strideHOperand));
190     NN_RET_CHECK(isOperandConstant(activationOperand));
191 
192     // get strides and activation
193     int32_t strideW = context->getConstantScalar<int32_t>(strideWOperand);
194     int32_t strideH = context->getConstantScalar<int32_t>(strideHOperand);
195     FusedActivationFunc activation = static_cast<FusedActivationFunc>(
196             context->getConstantScalar<int32_t>(activationOperand));
197 
198     // check for nchw
199     int isNchwIdx = baseOptionsIdx + kIsNchwOffset;
200     if (operation.inputs.size() > static_cast<uint32_t>(isNchwIdx)) {
201         const Operand& isNchwOperand = subgraph->operands[operation.inputs[isNchwIdx]];
202         NN_RET_CHECK(isOperandConstant(isNchwOperand));
203 
204         bool isNchw = context->getConstantScalar<bool>(isNchwOperand);
205         NN_RET_CHECK(!isNchw) << "TFLite does not support NCHW formatted input tensors";
206     }
207 
208     // dilations
209     int dilationWIdx = baseOptionsIdx + kDilationWOffset;
210     int dilationHIdx = baseOptionsIdx + kDilationHOffset;
211     // default dilation factors are 1
212     int32_t dilationW = 1;
213     int32_t dilationH = 1;
214     if (operation.inputs.size() > static_cast<uint32_t>(dilationWIdx)) {
215         const Operand& dilationWOperand = subgraph->operands[operation.inputs[dilationWIdx]];
216         NN_RET_CHECK(isOperandConstant(dilationWOperand));
217 
218         dilationW = context->getConstantScalar<int32_t>(dilationWOperand);
219     }
220     if (operation.inputs.size() > static_cast<uint32_t>(dilationHIdx)) {
221         const Operand& dilationHOperand = subgraph->operands[operation.inputs[dilationHIdx]];
222         NN_RET_CHECK(isOperandConstant(dilationHOperand));
223 
224         dilationH = context->getConstantScalar<int32_t>(dilationHOperand);
225     }
226 
227     flatbuffers::Offset<tflite::Conv2DOptions> optionsFlatbuffer = tflite::CreateConv2DOptions(
228             context->getBuilder(), padding, strideW, strideH,
229             NN_TRY(getTfliteActivation(activation)) /* fused_activation_function */, dilationW,
230             dilationH);
231     auto operatorFlatbuffer = tflite::CreateOperatorDirect(
232             context->getBuilder() /* builder */, opCodeIdx /* opcode_index */, &inputs /* inputs */,
233             &outputs /* outputs */,
234             tflite::BuiltinOptions::BuiltinOptions_Conv2DOptions /* builtin_options_type */,
235             optionsFlatbuffer.Union() /* builtin_options */);
236     context->addOperatorFlatbuffer(operatorFlatbuffer);
237 
238     return {};
239 }
240 
241 NN_REGISTER_OPERATION_CONVERTER(CONV_2D, Conv2DOperationConverter);
242 
243 }  // namespace nn
244 }  // namespace android