1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow/lite/delegates/gpu/common/model_builder_helper.h"
17 
18 #include <stddef.h>
19 #include <stdint.h>
20 #include <string.h>
21 
22 #include <any>
23 #include <limits>
24 #include <string>
25 #include <vector>
26 
27 #include <fp16.h>
28 #include "absl/strings/str_cat.h"
29 #include "absl/strings/str_join.h"
30 #include "tensorflow/lite/c/builtin_op_data.h"
31 #include "tensorflow/lite/c/common.h"
32 #include "tensorflow/lite/context_util.h"
33 #include "tensorflow/lite/delegates/gpu/common/data_type.h"
34 #include "tensorflow/lite/delegates/gpu/common/model.h"
35 #include "tensorflow/lite/delegates/gpu/common/operations.h"
36 #include "tensorflow/lite/delegates/gpu/common/shape.h"
37 #include "tensorflow/lite/delegates/gpu/common/status.h"
38 #include "tensorflow/lite/delegates/gpu/common/tensor.h"
39 #include "tensorflow/lite/kernels/kernel_util.h"
40 
41 namespace tflite {
42 namespace gpu {
43 namespace {
44 
45 // Creates a node that consumes output from the given node. Because output need
46 // to stay the same, newly created node will inherit the output from the given
47 // node, which will in turn get newly created copy of output. This is necessary
48 // to preserve reference consistency if another node was pointing at that
49 // output:
50 //   node(output)
51 // will turn into:
52 //   node(copy(output)) <- passthrough_node(output)
NewPassthroughNode(GraphFloat32 * graph,Node * node,const Value * output,Node ** passthru_node)53 absl::Status NewPassthroughNode(GraphFloat32* graph, Node* node,
54                                 const Value* output, Node** passthru_node) {
55   *passthru_node = graph->NewNode();
56   // Make copies for every output in the original node.
57   RETURN_IF_ERROR(graph->SetProducer((*passthru_node)->id, output->id));
58   Value* copy_output = graph->NewValue();
59   RETURN_IF_ERROR(graph->SetProducer(node->id, copy_output->id));
60   RETURN_IF_ERROR(graph->AddConsumer((*passthru_node)->id, copy_output->id));
61   copy_output->tensor = output->tensor;
62   copy_output->tensor.ref = -1;
63   return absl::OkStatus();
64 }
65 
66 }  // namespace
67 
GetNodeAndRegistration(TfLiteContext * context,int node_id,TfLiteNode ** tflite_node,TfLiteRegistration ** registration)68 absl::Status GetNodeAndRegistration(TfLiteContext* context, int node_id,
69                                     TfLiteNode** tflite_node,
70                                     TfLiteRegistration** registration) {
71   if (context->GetNodeAndRegistration(context, node_id, tflite_node,
72                                       registration) != kTfLiteOk) {
73     return absl::InvalidArgumentError(absl::StrCat(
74         "Couldn't get node and registration info for op: ", node_id));
75   }
76   return absl::OkStatus();
77 }
78 
ToDataType(TfLiteType type)79 DataType ToDataType(TfLiteType type) {
80   switch (type) {
81     case kTfLiteFloat32:
82       return DataType::FLOAT32;
83     case kTfLiteInt32:
84       return DataType::INT32;
85     case kTfLiteInt64:
86       return DataType::INT64;
87     case kTfLiteInt8:
88       return DataType::INT8;
89     case kTfLiteUInt8:
90       return DataType::UINT8;
91     default:
92       return DataType::UNKNOWN;
93   }
94 }
95 
ExtractTensorShape(const TfLiteTensor & tflite_tensor,BHWC * bhwc)96 absl::Status ExtractTensorShape(const TfLiteTensor& tflite_tensor, BHWC* bhwc) {
97   const TfLiteIntArray* dims = tflite_tensor.dims;
98   switch (dims->size) {
99     case 1:
100       // B layout
101       *bhwc = BHWC(dims->data[0], 1, 1, 1);
102       return absl::OkStatus();
103     case 2:
104       // BC layout
105       *bhwc = BHWC(dims->data[0], 1, 1, dims->data[1]);
106       return absl::OkStatus();
107     case 3:
108       // BWC layout
109       *bhwc = BHWC(dims->data[0], 1, dims->data[1], dims->data[2]);
110       return absl::OkStatus();
111     case 4:
112       // BHWC layout
113       *bhwc = BHWC(dims->data[0], dims->data[1], dims->data[2], dims->data[3]);
114       return absl::OkStatus();
115     default:
116       return absl::InvalidArgumentError(absl::StrCat(
117           "Tensor \"", tflite_tensor.name ? tflite_tensor.name : "nullptr",
118           "\" has bad input dims size: ", dims->size, "."));
119   }
120 }
121 
ExtractAxisFromIndex(const TfLiteTensor & tflite_tensor,int index,Axis * axis)122 absl::Status ExtractAxisFromIndex(const TfLiteTensor& tflite_tensor, int index,
123                                   Axis* axis) {
124   const TfLiteIntArray* dims = tflite_tensor.dims;
125   if (index == -1) {
126     index = dims->size - 1;
127   }
128   if (index < 0 || index >= dims->size) {
129     return absl::OutOfRangeError("Index for axis out of range");
130   }
131   std::vector<Axis> index_to_axis;
132   switch (dims->size) {
133     case 1:
134       // B layout
135       index_to_axis = {Axis::BATCH};
136       break;
137     case 2:
138       // BC layout
139       index_to_axis = {Axis::BATCH, Axis::CHANNELS};
140       break;
141     case 3:
142       // BWC layout
143       index_to_axis = {Axis::BATCH, Axis::WIDTH, Axis::CHANNELS};
144       break;
145     case 4:
146       // BHWC layout
147       index_to_axis = {Axis::BATCH, Axis::HEIGHT, Axis::WIDTH, Axis::CHANNELS};
148       break;
149     default:
150       return absl::UnavailableError("Unknown layout.");
151   }
152   *axis = index_to_axis[index];
153   return absl::OkStatus();
154 }
155 
ConvertTfLiteTensorToTensorRef(const TfLiteTensor & tflite_tensor,TensorRef<BHWC> * tensor_ref)156 absl::Status ConvertTfLiteTensorToTensorRef(const TfLiteTensor& tflite_tensor,
157                                             TensorRef<BHWC>* tensor_ref) {
158   tensor_ref->type = ToDataType(tflite_tensor.type);
159   return ExtractTensorShape(tflite_tensor, &tensor_ref->shape);
160 }
161 
PopulateQuantParams(const TfLiteTensor & tensor,QuantizationParams * quant_params)162 absl::Status PopulateQuantParams(const TfLiteTensor& tensor,
163                                  QuantizationParams* quant_params) {
164   const TfLiteQuantization& quant = tensor.quantization;
165   if (quant.type != TfLiteQuantizationType::kTfLiteAffineQuantization) {
166     return absl::InvalidArgumentError(
167         absl::StrCat("Tensor not quantized: ", std::string(tensor.name)));
168   }
169   const TfLiteAffineQuantization* params =
170       static_cast<const TfLiteAffineQuantization*>(quant.params);
171   if (params->scale->size > 1) {
172     return absl::InvalidArgumentError(
173         absl::StrCat("Non-constant per-channel quantized tensor: ",
174                      std::string(tensor.name)));
175   }
176   const float scale = params->scale->data[0];
177   const float zero_point = static_cast<float>(params->zero_point->data[0]);
178 
179   float qmin_value = 0;
180   float qmax_value = 0;
181   if (tensor.type == kTfLiteUInt8) {
182     qmin_value = static_cast<float>(std::numeric_limits<uint8_t>::min());
183     qmax_value = static_cast<float>(std::numeric_limits<uint8_t>::max());
184   } else if (tensor.type == kTfLiteInt8) {
185     qmin_value = static_cast<float>(std::numeric_limits<int8_t>::min());
186     qmax_value = static_cast<float>(std::numeric_limits<int8_t>::max());
187   } else {
188     return absl::InvalidArgumentError(absl::StrCat(
189         "Type invalid for quantized tensor: ", std::string(tensor.name)));
190   }
191   quant_params->min = scale * (static_cast<float>(qmin_value) - zero_point);
192   quant_params->max = scale * (static_cast<float>(qmax_value) - zero_point);
193   quant_params->scale = scale;
194 
195   return absl::OkStatus();
196 }
197 
GetNumberOfRuntimeInputsForNode(const TfLiteContext * context,const TfLiteNode * tflite_node)198 int GetNumberOfRuntimeInputsForNode(const TfLiteContext* context,
199                                     const TfLiteNode* tflite_node) {
200   int number_of_runtime_inputs = 0;
201   for (int i = 0; i < NumInputs(tflite_node); i++) {
202     const TfLiteTensor* tensor =
203         GetOptionalInputTensor(context, tflite_node, i);
204     if (tensor != nullptr && !IsConstantTensor(tensor)) {
205       number_of_runtime_inputs++;
206     }
207   }
208   return number_of_runtime_inputs;
209 }
210 
GetNumberOfConstInputsForNode(const TfLiteContext * context,const TfLiteNode * tflite_node)211 int GetNumberOfConstInputsForNode(const TfLiteContext* context,
212                                   const TfLiteNode* tflite_node) {
213   return NumInputs(tflite_node) -
214          GetNumberOfRuntimeInputsForNode(context, tflite_node);
215 }
216 
CheckInputsOutputs(const TfLiteContext * context,const TfLiteNode * tflite_node,int runtime_inputs,int outputs)217 absl::Status CheckInputsOutputs(const TfLiteContext* context,
218                                 const TfLiteNode* tflite_node,
219                                 int runtime_inputs, int outputs) {
220   const int runtime_inputs_from_model =
221       GetNumberOfRuntimeInputsForNode(context, tflite_node);
222   if (runtime_inputs_from_model != runtime_inputs) {
223     return absl::InternalError(absl::StrCat(
224         "Expected ", runtime_inputs, " runtime input tensor(s), but node has ",
225         runtime_inputs_from_model, " runtime input(s)."));
226   }
227   const int outputs_from_model = NumOutputs(tflite_node);
228   if (outputs_from_model != outputs) {
229     return absl::InternalError(absl::StrCat("Expected ", outputs,
230                                             " output tensor(s), but node has ",
231                                             outputs_from_model, " output(s)."));
232   }
233   return absl::OkStatus();
234 }
235 
CheckInputsConstsOutputs(const TfLiteContext * context,const TfLiteNode * tflite_node,int runtime_inputs,int const_inputs,int outputs)236 absl::Status CheckInputsConstsOutputs(const TfLiteContext* context,
237                                       const TfLiteNode* tflite_node,
238                                       int runtime_inputs, int const_inputs,
239                                       int outputs) {
240   const int const_inputs_from_model =
241       GetNumberOfConstInputsForNode(context, tflite_node);
242   if (const_inputs_from_model != const_inputs) {
243     return absl::InternalError(absl::StrCat(
244         "Expected ", const_inputs, " const input tensor(s), but node has ",
245         const_inputs_from_model, " const input(s)."));
246   }
247   return CheckInputsOutputs(context, tflite_node, runtime_inputs, outputs);
248 }
249 
ConvertFloat16ToFloat32(size_t num_elements,const uint16_t * src,float * dst)250 void ConvertFloat16ToFloat32(size_t num_elements, const uint16_t* src,
251                              float* dst) {
252   for (size_t i = 0; i < num_elements; i++) {
253     *dst++ = fp16_ieee_to_fp32_value(*src++);
254   }
255 }
256 
257 template <>
CreateVectorCopyData(const TfLiteTensor & tensor,float * tensor_data)258 absl::Status CreateVectorCopyData<float>(const TfLiteTensor& tensor,
259                                          float* tensor_data) {
260   switch (tensor.type) {
261     case kTfLiteFloat32:
262       std::memcpy(tensor_data, tensor.data.f, tensor.bytes);
263       break;
264     case kTfLiteFloat16:
265       ConvertFloat16ToFloat32(
266           NumElements(&tensor),
267           reinterpret_cast<uint16_t const*>(tensor.data.f16), tensor_data);
268       break;
269     case kTfLiteInt8:
270       DequantizeConstantTensor(tensor, tensor.data.int8, tensor_data);
271       break;
272     case kTfLiteUInt8:
273       DequantizeConstantTensor(tensor, tensor.data.uint8, tensor_data);
274       break;
275     case kTfLiteInt32:
276       DequantizeConstantTensor(tensor, tensor.data.i32, tensor_data);
277       break;
278     default:
279       return absl::InvalidArgumentError(
280           "Unsupported data type for float32 tensor");
281   }
282   return absl::OkStatus();
283 }
284 
GetDimensionString(const TfLiteIntArray * dimensions)285 const std::string GetDimensionString(const TfLiteIntArray* dimensions) {
286   return absl::StrJoin(TfLiteIntArrayView(dimensions), "x");
287 }
288 
SetAllDimensions(const TfLiteIntArray * dimensions,Scalar * shape)289 absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, Scalar* shape) {
290   if (dimensions->size < 0) {
291     return absl::InvalidArgumentError("Invalid Scalar dimensions");
292   }
293   for (int i = 0; i < dimensions->size; ++i) {
294     if (dimensions->data[i] != 1) {
295       return absl::InvalidArgumentError(absl::StrCat(
296           GetDimensionString(dimensions), "  cannot be reduced to scalar."));
297     }
298   }
299   shape->v = 1;
300   return absl::OkStatus();
301 }
302 
CheckIfLinearConvertible(const TfLiteIntArray * dimensions)303 absl::Status CheckIfLinearConvertible(const TfLiteIntArray* dimensions) {
304   if (dimensions->size <= 0) {
305     return absl::InvalidArgumentError("Dimension is empty.");
306   }
307   for (int i = 0; i < dimensions->size - 1; ++i) {
308     if (dimensions->data[i] != 1) {
309       return absl::InvalidArgumentError(absl::StrCat(
310           GetDimensionString(dimensions), "  cannot be reduced to linear."));
311     }
312   }
313   return absl::OkStatus();
314 }
315 
SetAllDimensions(const TfLiteIntArray * dimensions,Linear * shape)316 absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, Linear* shape) {
317   RETURN_IF_ERROR(CheckIfLinearConvertible(dimensions));
318   shape->v = dimensions->data[dimensions->size - 1];
319   return absl::OkStatus();
320 }
321 
SetAllDimensions(const TfLiteIntArray * dimensions,HWC * shape)322 absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, HWC* shape) {
323   if (dimensions->size == 3) {
324     shape->h = dimensions->data[0];
325     shape->w = dimensions->data[1];
326     shape->c = dimensions->data[2];
327     return absl::OkStatus();
328   }
329   if (dimensions->size == 4) {
330     if (dimensions->data[0] != 1) {
331       return absl::UnimplementedError("Batch size is not equal to 1.");
332     }
333     shape->h = dimensions->data[1];
334     shape->w = dimensions->data[2];
335     shape->c = dimensions->data[3];
336     return absl::OkStatus();
337   }
338   return absl::InvalidArgumentError(
339       absl::StrCat("Expected a 3D tensor of shape HxWxC or a 4D tensor of "
340                    "shape 1xHxWxC but got ",
341                    GetDimensionString(dimensions)));
342 }
343 
SetAllDimensions(const TfLiteIntArray * dimensions,HW * shape)344 absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, HW* shape) {
345   if (dimensions->size != 2) {
346     return absl::InvalidArgumentError(
347         absl::StrCat("Expected a 2D tensor of shape HxW but got ",
348                      GetDimensionString(dimensions)));
349   }
350   shape->h = dimensions->data[0];
351   shape->w = dimensions->data[1];
352   return absl::OkStatus();
353 }
354 
SetAllDimensions(const TfLiteIntArray * dimensions,OHWI * shape)355 absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, OHWI* shape) {
356   if (dimensions->size != 4) {
357     return absl::InvalidArgumentError(
358         absl::StrCat("Expected a 4D tensor of shape OxHxWxI but got ",
359                      GetDimensionString(dimensions)));
360   }
361   shape->o = dimensions->data[0];
362   shape->h = dimensions->data[1];
363   shape->w = dimensions->data[2];
364   shape->i = dimensions->data[3];
365   return absl::OkStatus();
366 }
367 
SetAllDimensions(const TfLiteIntArray * dimensions,BHWC * shape)368 absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, BHWC* shape) {
369   if (dimensions->size != 4) {
370     return absl::InvalidArgumentError(
371         absl::StrCat("Expected a 4D tensor of shape BxHxWxC but got ",
372                      GetDimensionString(dimensions)));
373   }
374   shape->b = dimensions->data[0];
375   shape->h = dimensions->data[1];
376   shape->w = dimensions->data[2];
377   shape->c = dimensions->data[3];
378   return absl::OkStatus();
379 }
380 
IsActivationSupported(TfLiteFusedActivation fused_activation)381 absl::Status IsActivationSupported(TfLiteFusedActivation fused_activation) {
382   switch (fused_activation) {
383     case kTfLiteActNone:
384     case kTfLiteActRelu:
385     case kTfLiteActReluN1To1:
386     case kTfLiteActRelu6:
387     case kTfLiteActTanh:
388     case kTfLiteActSigmoid:
389       return absl::OkStatus();
390     case kTfLiteActSignBit:
391       return absl::UnimplementedError(
392           "TfLiteFusedActivation.kTfLiteActSignBit");
393 
394       // Do not add default; we want compilation error rather than run-time
395       // error.
396   }
397 }
398 
399 // If there is fused activation present, then there will be another node created
400 // that will have identical output as the given node. New operation node will
401 // depend on the given node output.
MaybeFuseActivation(TfLiteFusedActivation fused_activation,GraphFloat32 * graph,Node * node)402 absl::Status MaybeFuseActivation(TfLiteFusedActivation fused_activation,
403                                  GraphFloat32* graph, Node* node) {
404   const auto outputs = graph->FindOutputs(node->id);
405   if (outputs.size() != 1) {
406     return absl::InternalError("Number of outputs != 1");
407   }
408   switch (fused_activation) {
409     case kTfLiteActNone:
410       // Nothing to do here
411       return absl::OkStatus();
412     case kTfLiteActRelu:
413     case kTfLiteActReluN1To1:
414     case kTfLiteActRelu6: {
415       ReLUAttributes attr;
416       attr.clip = fused_activation == kTfLiteActRelu
417                       ? 0.0f
418                       : (fused_activation == kTfLiteActReluN1To1 ? 1.0f : 6.0f);
419       Node* activation_node;
420       RETURN_IF_ERROR(
421           NewPassthroughNode(graph, node, outputs[0], &activation_node));
422       activation_node->operation.type = ToString(OperationType::RELU);
423       activation_node->operation.attributes = attr;
424       return absl::OkStatus();
425     }
426     case kTfLiteActTanh: {
427       Node* activation_node;
428       RETURN_IF_ERROR(
429           NewPassthroughNode(graph, node, outputs[0], &activation_node));
430       activation_node->operation.type = ToString(OperationType::TANH);
431       return absl::OkStatus();
432     }
433     case kTfLiteActSigmoid: {
434       Node* activation_node;
435       RETURN_IF_ERROR(
436           NewPassthroughNode(graph, node, outputs[0], &activation_node));
437       activation_node->operation.type = ToString(OperationType::SIGMOID);
438       return absl::OkStatus();
439     } break;
440     default:
441       return absl::NotFoundError(
442           absl::StrCat("Unsupported fused activation: ", fused_activation));
443   }
444 }
445 
446 }  // namespace gpu
447 }  // namespace tflite
448