1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Operations"
18 
19 #include "ResizeImageOps.h"
20 
21 #include <algorithm>
22 #include <functional>
23 #include <vector>
24 
25 #include "OperationResolver.h"
26 #include "Tracing.h"
27 #include "nnapi/Validation.h"
28 
29 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
30 #pragma clang diagnostic push
31 #pragma clang diagnostic ignored "-Wunused-parameter"
32 #pragma clang diagnostic ignored "-Wsign-compare"
33 #include <tensorflow/lite/kernels/internal/reference/reference_ops.h>
34 #pragma clang diagnostic pop
35 
36 #include "CpuOperationUtils.h"
37 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
38 
39 namespace android {
40 namespace nn {
41 
42 namespace resize_image {
43 
44 #ifdef NN_INCLUDE_CPU_IMPLEMENTATION
45 namespace {
46 
scaleHalfPixel(const int x,const float scale)47 inline float scaleHalfPixel(const int x, const float scale) {
48     return (static_cast<float>(x) + 0.5f) * scale;
49 }
50 
scaleLegacy(const int x,const float scale)51 inline float scaleLegacy(const int x, const float scale) {
52     return static_cast<float>(x) * scale;
53 }
54 
calculateResizeScale(int32_t inSize,int32_t outSize,bool alignCorners)55 inline float calculateResizeScale(int32_t inSize, int32_t outSize, bool alignCorners) {
56     return (alignCorners && outSize > 1) ? (inSize - 1) / static_cast<float>(outSize - 1)
57                                          : inSize / static_cast<float>(outSize);
58 }
59 
60 template <typename T>
resizeNearestNeighbor(const T * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)61 bool resizeNearestNeighbor(const T* inputData, const Shape& inputShape, bool alignCorners,
62                            bool halfPixelCenters, T* outputData, const Shape& outputShape) {
63     const int batchSize = getSizeOfDimension(inputShape, 0);
64     const int inHeight = getSizeOfDimension(inputShape, 1);
65     const int inWidth = getSizeOfDimension(inputShape, 2);
66     const int channels = getSizeOfDimension(inputShape, 3);
67     const int outHeight = getSizeOfDimension(outputShape, 1);
68     const int outWidth = getSizeOfDimension(outputShape, 2);
69 
70     const float heightScale = calculateResizeScale(inHeight, outHeight, alignCorners);
71     const float widthScale = calculateResizeScale(inWidth, outWidth, alignCorners);
72 
73     const std::function<float(const int, const float)> scaler =
74             halfPixelCenters ? scaleHalfPixel : scaleLegacy;
75 
76     for (int b = 0; b < batchSize; ++b) {
77         for (int y = 0; y < outHeight; ++y) {
78             int inY = std::min((alignCorners) ? static_cast<int>(roundf(scaler(y, heightScale)))
79                                               : static_cast<int>(floorf(scaler(y, heightScale))),
80                                inHeight - 1);
81             if (halfPixelCenters) {
82                 inY = std::max(static_cast<int>(0), inY);
83             }
84             for (int x = 0; x < outWidth; ++x) {
85                 int inX = std::min((alignCorners) ? static_cast<int>(roundf(scaler(x, widthScale)))
86                                                   : static_cast<int>(floorf(scaler(x, widthScale))),
87                                    inWidth - 1);
88                 if (halfPixelCenters) {
89                     inX = std::max(static_cast<int>(0), inX);
90                 }
91                 std::copy_n(inputData + b * inHeight * inWidth * channels +
92                                     inY * inWidth * channels + inX * channels,
93                             channels,
94                             outputData + b * outHeight * outWidth * channels +
95                                     y * outWidth * channels + x * channels);
96             }
97         }
98     }
99 
100     return true;
101 }
102 
103 template <typename T>
resizeImageOpNhwc(OperationType opType,const T * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)104 bool resizeImageOpNhwc(OperationType opType, const T* inputData, const Shape& inputShape,
105                        bool alignCorners, bool halfPixelCenters, T* outputData,
106                        const Shape& outputShape) {
107     NNTRACE_TRANS("resizeImageOpNhwc");
108     int32_t height = static_cast<int32_t>(getSizeOfDimension(outputShape, 1));
109     int32_t width = static_cast<int32_t>(getSizeOfDimension(outputShape, 2));
110     // We have to fake a tensor here, to satisfy tflite implementation.
111     int32_t outDimData[2] = {height, width};
112     Shape outDimShape;
113     outDimShape.dimensions = {2};
114 
115     if (opType == OperationType::RESIZE_BILINEAR) {
116         NNTRACE_COMP_SWITCH("optimized_ops::ResizeBilinear");
117         tflite::reference_ops::ResizeBilinear(
118                 {.align_corners = alignCorners, .half_pixel_centers = halfPixelCenters},
119                 convertShapeToTflshape(inputShape), inputData, convertShapeToTflshape(outDimShape),
120                 outDimData, convertShapeToTflshape(outputShape), outputData);
121     } else if (opType == OperationType::RESIZE_NEAREST_NEIGHBOR) {
122         // Align corners = true is not supported.
123         NNTRACE_COMP_SWITCH("ResizeNearestNeighbor");
124         resizeNearestNeighbor(inputData, inputShape, alignCorners, halfPixelCenters, outputData,
125                               outputShape);
126     }
127     return true;
128 }
129 
130 template <>
resizeImageOpNhwc(OperationType opType,const _Float16 * inputData,const Shape & inputShape,bool alignCorners,bool halfPixelCenters,_Float16 * outputData,const Shape & outputShape)131 bool resizeImageOpNhwc<_Float16>(OperationType opType, const _Float16* inputData,
132                                  const Shape& inputShape, bool alignCorners, bool halfPixelCenters,
133                                  _Float16* outputData, const Shape& outputShape) {
134     NNTRACE_TRANS("resizeImageOpNhwcFloat16");
135     std::vector<float> inputData_float32(getNumberOfElements(inputShape));
136     convertFloat16ToFloat32(inputData, &inputData_float32);
137     std::vector<float> outputData_float32(getNumberOfElements(outputShape));
138     NN_RET_CHECK(resizeImageOpNhwc(opType, inputData_float32.data(), inputShape, alignCorners,
139                                    halfPixelCenters, outputData_float32.data(), outputShape));
140     convertFloat32ToFloat16(outputData_float32, outputData);
141     return true;
142 }
143 
144 template <typename T>
resizeImageOp(OperationType opType,const T * inputData,const Shape & inputShape,bool useNchw,bool alignCorners,bool halfPixelCenters,T * outputData,const Shape & outputShape)145 bool resizeImageOp(OperationType opType, const T* inputData, const Shape& inputShape, bool useNchw,
146                    bool alignCorners, bool halfPixelCenters, T* outputData,
147                    const Shape& outputShape) {
148     InputWithLayout<T> input(useNchw);
149     OutputWithLayout<T> output(useNchw);
150     NN_RET_CHECK(input.initialize(inputData, inputShape));
151     NN_RET_CHECK(output.initialize(outputData, outputShape));
152     NN_RET_CHECK(resizeImageOpNhwc(opType, input.getNhwcBuffer(), input.getNhwcShape(),
153                                    alignCorners, halfPixelCenters, output.getNhwcBuffer(),
154                                    output.getNhwcShape()));
155     NN_RET_CHECK(output.commit());
156     return true;
157 }
158 
getOptionalScalar(const IOperationExecutionContext * context,uint32_t scalarIndex)159 inline bool getOptionalScalar(const IOperationExecutionContext* context, uint32_t scalarIndex) {
160     bool scalarValue = false;
161     if (context->getNumInputs() > scalarIndex) {
162         scalarValue = context->getInputValue<bool>(scalarIndex);
163     }
164     return scalarValue;
165 }
166 
167 }  // namespace
168 
prepare(OperationType opType,IOperationExecutionContext * context)169 bool prepare(OperationType opType, IOperationExecutionContext* context) {
170     Shape input = context->getInputShape(kInputTensor);
171     NN_RET_CHECK_EQ(getNumberOfDimensions(input), 4u);
172     [[maybe_unused]] const auto numInputs = context->getNumInputs();
173     const bool useNchw = getOptionalScalar(context, kLayoutScalar);
174     const bool alignCorners = getOptionalScalar(context, kAlignCornersScalar);
175     const bool halfPixelCenters = getOptionalScalar(context, kHalfPixelCentersScalar);
176 
177     NN_RET_CHECK(!halfPixelCenters || (halfPixelCenters && !alignCorners));
178 
179     // Only batches can be zero.
180     uint32_t batches = getSizeOfDimension(input, 0);
181     uint32_t inHeight = getSizeOfDimension(input, useNchw ? 2 : 1);
182     uint32_t inWidth = getSizeOfDimension(input, useNchw ? 3 : 2);
183     uint32_t channels = getSizeOfDimension(input, useNchw ? 1 : 3);
184     NN_RET_CHECK_GT(inHeight, 0u);
185     NN_RET_CHECK_GT(inWidth, 0u);
186     NN_RET_CHECK_GT(channels, 0u);
187 
188     int32_t height, width;
189     auto scalarType = context->getInputType(kOutputHeightParamScalar);
190     if (scalarType == OperandType::INT32) {
191         height = context->getInputValue<int32_t>(kOutputHeightParamScalar);
192         width = context->getInputValue<int32_t>(kOutputWidthParamScalar);
193     } else if (scalarType == OperandType::FLOAT32) {
194         height = std::floor(static_cast<float>(inHeight) *
195                             context->getInputValue<float>(kOutputHeightParamScalar));
196         width = std::floor(static_cast<float>(inWidth) *
197                            context->getInputValue<float>(kOutputWidthParamScalar));
198     } else if (scalarType == OperandType::FLOAT16) {
199         height = std::floor(
200                 static_cast<float>(inHeight) *
201                 static_cast<float>(context->getInputValue<_Float16>(kOutputHeightParamScalar)));
202         width = std::floor(
203                 static_cast<float>(inWidth) *
204                 static_cast<float>(context->getInputValue<_Float16>(kOutputWidthParamScalar)));
205     } else {
206         NN_RET_CHECK_FAIL() << "Unsupported scalar type for operation " << opType;
207     }
208     NN_RET_CHECK_GT(height, 0);
209     NN_RET_CHECK_GT(width, 0);
210 
211     Shape output = input;
212     if (useNchw) {
213         output.dimensions = {batches, channels, (uint32_t)height, (uint32_t)width};
214     } else {
215         output.dimensions = {batches, (uint32_t)height, (uint32_t)width, channels};
216     }
217     return context->setOutputShape(kOutputTensor, output);
218 }
219 
execute(OperationType opType,IOperationExecutionContext * context)220 bool execute(OperationType opType, IOperationExecutionContext* context) {
221     // Bypass execution in the case of zero-sized input.
222     if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
223 
224     const bool useNchw = getOptionalScalar(context, kLayoutScalar);
225     const bool alignCorners = getOptionalScalar(context, kAlignCornersScalar);
226     const bool halfPixelCenters = getOptionalScalar(context, kHalfPixelCentersScalar);
227 
228     switch (context->getInputType(kInputTensor)) {
229         case OperandType::TENSOR_FLOAT16:
230             return resizeImageOp(opType, context->getInputBuffer<_Float16>(kInputTensor),
231                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
232                                  halfPixelCenters,
233                                  context->getOutputBuffer<_Float16>(kOutputTensor),
234                                  context->getOutputShape(kOutputTensor));
235         case OperandType::TENSOR_FLOAT32:
236             return resizeImageOp(opType, context->getInputBuffer<float>(kInputTensor),
237                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
238                                  halfPixelCenters, context->getOutputBuffer<float>(kOutputTensor),
239                                  context->getOutputShape(kOutputTensor));
240         case OperandType::TENSOR_QUANT8_ASYMM:
241             return resizeImageOp(opType, context->getInputBuffer<uint8_t>(kInputTensor),
242                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
243                                  halfPixelCenters, context->getOutputBuffer<uint8_t>(kOutputTensor),
244                                  context->getOutputShape(kOutputTensor));
245         case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
246             return resizeImageOp(opType, context->getInputBuffer<int8_t>(kInputTensor),
247                                  context->getInputShape(kInputTensor), useNchw, alignCorners,
248                                  halfPixelCenters, context->getOutputBuffer<int8_t>(kOutputTensor),
249                                  context->getOutputShape(kOutputTensor));
250 
251         default:
252             NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << opType;
253     }
254 }
255 #endif  // NN_INCLUDE_CPU_IMPLEMENTATION
256 
257 }  // namespace resize_image
258 
259 using std::placeholders::_1;
260 
261 NN_REGISTER_OPERATION_DEFAULT_VALIDATION(
262         RESIZE_BILINEAR, std::bind(resize_image::prepare, OperationType::RESIZE_BILINEAR, _1),
263         std::bind(resize_image::execute, OperationType::RESIZE_BILINEAR, _1),
264         .allowZeroSizedInput = true);
265 
266 NN_REGISTER_OPERATION_DEFAULT_VALIDATION(RESIZE_NEAREST_NEIGHBOR,
267                                          std::bind(resize_image::prepare,
268                                                    OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
269                                          std::bind(resize_image::execute,
270                                                    OperationType::RESIZE_NEAREST_NEIGHBOR, _1),
271                                          .allowZeroSizedInput = true);
272 
273 }  // namespace nn
274 }  // namespace android
275