1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include "tensorflow/lite/kernels/test_util.h"
16
17 #include <stddef.h>
18 #include <stdint.h>
19
20 #include <algorithm>
21 #include <complex>
22 #include <functional>
23 #include <map>
24 #include <memory>
25 #include <optional>
26 #include <string>
27 #include <tuple>
28 #include <utility>
29 #include <vector>
30
31 #include <gmock/gmock.h>
32 #include <gtest/gtest.h>
33 #include "flatbuffers/flatbuffers.h" // from @flatbuffers
34 #include "tensorflow/core/platform/logging.h"
35 #include "tensorflow/lite/c/common.h"
36 #include "tensorflow/lite/core/api/op_resolver.h"
37 #include "tensorflow/lite/core/subgraph.h"
38 #include "tensorflow/lite/delegates/nnapi/acceleration_test_util.h"
39 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
40 #include "tensorflow/lite/interpreter.h"
41 #include "tensorflow/lite/kernels/acceleration_test_util.h"
42 #include "tensorflow/lite/kernels/register.h"
43 #include "tensorflow/lite/kernels/test_delegate_providers.h"
44 #include "tensorflow/lite/model.h"
45 #include "tensorflow/lite/nnapi/nnapi_implementation.h"
46 #include "tensorflow/lite/schema/schema_conversion_utils.h"
47 #include "tensorflow/lite/schema/schema_generated.h"
48 #include "tensorflow/lite/string_type.h"
49 #include "tensorflow/lite/string_util.h"
50 #include "tensorflow/lite/tools/logging.h"
51 #include "tensorflow/lite/tools/versioning/op_version.h"
52 #include "tensorflow/lite/version.h"
53
54 namespace tflite {
55
56 using ::testing::FloatNear;
57 using ::testing::Matcher;
58
ArrayFloatNear(const std::vector<float> & values,float max_abs_error)59 std::vector<Matcher<float>> ArrayFloatNear(const std::vector<float>& values,
60 float max_abs_error) {
61 std::vector<Matcher<float>> matchers;
62 matchers.reserve(values.size());
63 for (const float& v : values) {
64 matchers.emplace_back(FloatNear(v, max_abs_error));
65 }
66 return matchers;
67 }
68
ArrayComplex64Near(const std::vector<std::complex<float>> & values,float max_abs_error)69 std::vector<Matcher<std::complex<float>>> ArrayComplex64Near(
70 const std::vector<std::complex<float>>& values, float max_abs_error) {
71 std::vector<Matcher<std::complex<float>>> matchers;
72 matchers.reserve(values.size());
73 for (const std::complex<float>& v : values) {
74 matchers.emplace_back(
75 AllOf(::testing::Property(&std::complex<float>::real,
76 FloatNear(v.real(), max_abs_error)),
77 ::testing::Property(&std::complex<float>::imag,
78 FloatNear(v.imag(), max_abs_error))));
79 }
80 return matchers;
81 }
82
AddInput(const TensorData & t)83 int SingleOpModel::AddInput(const TensorData& t) {
84 int id = 0;
85 if (t.per_channel_quantization) {
86 id = AddTensorPerChannelQuant(t);
87 } else {
88 id = AddTensor<float>(t, {});
89 }
90 inputs_.push_back(id);
91 return id;
92 }
93
AddVariableInput(const TensorData & t)94 int SingleOpModel::AddVariableInput(const TensorData& t) {
95 int id = 0;
96 if (t.per_channel_quantization) {
97 id = AddTensorPerChannelQuant(t);
98 } else {
99 id = AddTensor<float>(t, {}, true);
100 }
101 inputs_.push_back(id);
102 return id;
103 }
104
AddIntermediate(TensorType type,const std::vector<float> & scale,const std::vector<int64_t> & zero_point)105 int SingleOpModel::AddIntermediate(TensorType type,
106 const std::vector<float>& scale,
107 const std::vector<int64_t>& zero_point) {
108 // Currently supports only int16 intermediate types.
109 // TODO(jianlijianli): make use of the type.
110 int id = tensors_.size();
111 flatbuffers::Offset<QuantizationParameters> q_params =
112 CreateQuantizationParameters(builder_, /*min=*/0, /*max=*/0,
113 builder_.CreateVector<float>(scale),
114 builder_.CreateVector<int64_t>(zero_point));
115 tensors_.push_back(CreateTensor(builder_, builder_.CreateVector<int>({}),
116 type,
117 /*buffer=*/0,
118 /*name=*/0, q_params, false));
119 intermediates_.push_back(id);
120 return id;
121 }
122
AddNullInput()123 int SingleOpModel::AddNullInput() {
124 int id = kTfLiteOptionalTensor;
125 inputs_.push_back(id);
126 return id;
127 }
128
AddOutput(const TensorData & t)129 int SingleOpModel::AddOutput(const TensorData& t) {
130 int id = AddTensor<float>(t, {});
131 outputs_.push_back(id);
132 return id;
133 }
134
SetBuiltinOp(BuiltinOperator type,BuiltinOptions builtin_options_type,flatbuffers::Offset<void> builtin_options)135 void SingleOpModel::SetBuiltinOp(BuiltinOperator type,
136 BuiltinOptions builtin_options_type,
137 flatbuffers::Offset<void> builtin_options) {
138 opcodes_.push_back(CreateOperatorCode(builder_, type, 0, 0));
139 operators_.push_back(CreateOperator(
140 builder_, /*opcode_index=*/0, builder_.CreateVector<int32_t>(inputs_),
141 builder_.CreateVector<int32_t>(outputs_), builtin_options_type,
142 builtin_options,
143 /*custom_options=*/0, CustomOptionsFormat_FLEXBUFFERS, 0,
144 builder_.CreateVector<int32_t>(intermediates_)));
145 }
146
SetCustomOp(const string & name,const std::vector<uint8_t> & custom_option,const std::function<TfLiteRegistration * ()> & registration)147 void SingleOpModel::SetCustomOp(
148 const string& name, const std::vector<uint8_t>& custom_option,
149 const std::function<TfLiteRegistration*()>& registration) {
150 custom_registrations_[name] = registration;
151 opcodes_.push_back(
152 CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
153 operators_.push_back(CreateOperator(
154 builder_, /*opcode_index=*/0, builder_.CreateVector<int32_t>(inputs_),
155 builder_.CreateVector<int32_t>(outputs_), BuiltinOptions_NONE, 0,
156 builder_.CreateVector<uint8_t>(custom_option),
157 CustomOptionsFormat_FLEXBUFFERS));
158 }
159
AllocateAndDelegate(bool apply_delegate)160 void SingleOpModel::AllocateAndDelegate(bool apply_delegate) {
161 CHECK(interpreter_->AllocateTensors() == kTfLiteOk)
162 << "Cannot allocate tensors";
163 interpreter_->ResetVariableTensors();
164
165 // In some rare cases a test may need to postpone modifying the graph with
166 // a delegate, e.g. if tensors are not fully specified. In such cases the
167 // test has to explicitly call ApplyDelegate() when necessary.
168 if (apply_delegate) ApplyDelegate();
169 }
170
BuildInterpreter(std::vector<std::vector<int>> input_shapes,int num_threads,bool allow_fp32_relax_to_fp16,bool apply_delegate,bool allocate_and_delegate)171 void SingleOpModel::BuildInterpreter(std::vector<std::vector<int>> input_shapes,
172 int num_threads,
173 bool allow_fp32_relax_to_fp16,
174 bool apply_delegate,
175 bool allocate_and_delegate) {
176 auto opcodes = builder_.CreateVector(opcodes_);
177 auto operators = builder_.CreateVector(operators_);
178 auto tensors = builder_.CreateVector(tensors_);
179 auto inputs = builder_.CreateVector<int32_t>(inputs_);
180 auto outputs = builder_.CreateVector<int32_t>(outputs_);
181 // Create a single subgraph
182 std::vector<flatbuffers::Offset<SubGraph>> subgraphs;
183 auto subgraph = CreateSubGraph(builder_, tensors, inputs, outputs, operators);
184 subgraphs.push_back(subgraph);
185 auto subgraphs_flatbuffer = builder_.CreateVector(subgraphs);
186
187 auto buffers = builder_.CreateVector(buffers_);
188 auto description = builder_.CreateString("programmatic model");
189 builder_.Finish(CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
190 subgraphs_flatbuffer, description, buffers));
191
192 uint8_t* buffer_pointer = builder_.GetBufferPointer();
193 UpdateOpVersion(buffer_pointer);
194
195 if (!resolver_) {
196 MutableOpResolver* resolver =
197 apply_delegate
198 ? new ops::builtin::BuiltinOpResolver()
199 : new ops::builtin::BuiltinOpResolverWithoutDefaultDelegates();
200 for (const auto& reg : custom_registrations_) {
201 resolver->AddCustom(reg.first.data(), reg.second());
202 }
203 resolver_ = std::unique_ptr<OpResolver>(resolver);
204 }
205 CHECK(InterpreterBuilder(GetModel(buffer_pointer), *resolver_)(
206 &interpreter_, num_threads) == kTfLiteOk);
207
208 CHECK(interpreter_ != nullptr);
209
210 for (size_t i = 0; i < input_shapes.size(); ++i) {
211 const int input_idx = interpreter_->inputs()[i];
212 if (input_idx == kTfLiteOptionalTensor) continue;
213 const auto& shape = input_shapes[i];
214 if (shape.empty()) continue;
215 CHECK(interpreter_->ResizeInputTensor(input_idx, shape) == kTfLiteOk);
216 }
217
218 interpreter_->SetAllowFp16PrecisionForFp32(allow_fp32_relax_to_fp16);
219
220 if (allocate_and_delegate) {
221 AllocateAndDelegate(apply_delegate);
222 }
223 }
224
ApplyDelegate()225 TfLiteStatus SingleOpModel::ApplyDelegate() {
226 if (delegate_) {
227 TFLITE_LOG(WARN) << "Having a manually-set TfLite delegate, and bypassing "
228 "KernelTestDelegateProviders";
229 TF_LITE_ENSURE_STATUS(interpreter_->ModifyGraphWithDelegate(delegate_));
230 ++num_applied_delegates_;
231 } else {
232 auto* delegate_providers = tflite::KernelTestDelegateProviders::Get();
233 // Most TFLite NNAPI delegation tests have been written to run against the
234 // NNAPI CPU path. We'll enable that for tests. However, need to first check
235 // if the parameter is present - it will not be if the NNAPI delegate
236 // provider is not linked into the test.
237 if (delegate_providers->ConstParams().HasParam("disable_nnapi_cpu")) {
238 delegate_providers->MutableParams()->Set("disable_nnapi_cpu", false);
239 }
240 for (auto& one : delegate_providers->CreateAllDelegates()) {
241 // The raw ptr always points to the actual TfLiteDegate object.
242 auto* delegate_raw_ptr = one.get();
243 TF_LITE_ENSURE_STATUS(
244 interpreter_->ModifyGraphWithDelegate(std::move(one)));
245 // Note: 'delegate_' is always set to the last successfully applied one.
246 delegate_ = delegate_raw_ptr;
247 ++num_applied_delegates_;
248 }
249 }
250 return kTfLiteOk;
251 }
252
Invoke()253 void SingleOpModel::Invoke() { ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk); }
254
InvokeUnchecked()255 TfLiteStatus SingleOpModel::InvokeUnchecked() { return interpreter_->Invoke(); }
256
BuildInterpreter(std::vector<std::vector<int>> input_shapes)257 void SingleOpModel::BuildInterpreter(
258 std::vector<std::vector<int>> input_shapes) {
259 BuildInterpreter(input_shapes, /*num_threads=*/-1,
260 /*allow_fp32_relax_to_fp16=*/false,
261 /*apply_delegate=*/true, /*allocate_and_delegate=*/true);
262 }
263
264 // static
GetForceUseNnapi()265 bool SingleOpModel::GetForceUseNnapi() {
266 const auto& delegate_params =
267 tflite::KernelTestDelegateProviders::Get()->ConstParams();
268 // It's possible this library isn't linked with the nnapi delegate provider
269 // lib.
270 return delegate_params.HasParam("use_nnapi") &&
271 delegate_params.Get<bool>("use_nnapi");
272 }
273
GetTensorSize(int index) const274 int32_t SingleOpModel::GetTensorSize(int index) const {
275 TfLiteTensor* t = interpreter_->tensor(index);
276 CHECK(t);
277 int total_size = 1;
278 for (int i = 0; i < t->dims->size; ++i) {
279 total_size *= t->dims->data[i];
280 }
281 return total_size;
282 }
283
284 template <>
ExtractVector(int index) const285 std::vector<string> SingleOpModel::ExtractVector(int index) const {
286 TfLiteTensor* tensor_ptr = interpreter_->tensor(index);
287 CHECK(tensor_ptr != nullptr);
288 const int num_strings = GetStringCount(tensor_ptr);
289 std::vector<string> result;
290 result.reserve(num_strings);
291 for (int i = 0; i < num_strings; ++i) {
292 const auto str = GetString(tensor_ptr, i);
293 result.emplace_back(str.str, str.len);
294 }
295 return result;
296 }
297
298 namespace {
299
300 // Returns the number of partitions associated, as result of a call to
301 // ModifyGraphWithDelegate, to the given delegate.
CountPartitionsDelegatedTo(Subgraph * subgraph,const TfLiteDelegate * delegate)302 int CountPartitionsDelegatedTo(Subgraph* subgraph,
303 const TfLiteDelegate* delegate) {
304 return std::count_if(
305 subgraph->nodes_and_registration().begin(),
306 subgraph->nodes_and_registration().end(),
307 [delegate](
308 std::pair<TfLiteNode, TfLiteRegistration> node_and_registration) {
309 return node_and_registration.first.delegate == delegate;
310 });
311 }
312
313 // Returns the number of partitions associated, as result of a call to
314 // ModifyGraphWithDelegate, to the given delegate.
CountPartitionsDelegatedTo(Interpreter * interpreter,const TfLiteDelegate * delegate)315 int CountPartitionsDelegatedTo(Interpreter* interpreter,
316 const TfLiteDelegate* delegate) {
317 int result = 0;
318 for (int i = 0; i < interpreter->subgraphs_size(); i++) {
319 Subgraph* subgraph = interpreter->subgraph(i);
320
321 result += CountPartitionsDelegatedTo(subgraph, delegate);
322 }
323
324 return result;
325 }
326
327 // Returns the number of nodes that will be executed on the CPU
CountPartitionsExecutedByCpuKernel(const Interpreter * interpreter)328 int CountPartitionsExecutedByCpuKernel(const Interpreter* interpreter) {
329 int result = 0;
330 for (int node_idx : interpreter->execution_plan()) {
331 TfLiteNode node;
332 TfLiteRegistration reg;
333 std::tie(node, reg) = *(interpreter->node_and_registration(node_idx));
334
335 if (node.delegate == nullptr) {
336 ++result;
337 }
338 }
339
340 return result;
341 }
342
343 } // namespace
344
ExpectOpAcceleratedWithNnapi(const std::string & test_id)345 void SingleOpModel::ExpectOpAcceleratedWithNnapi(const std::string& test_id) {
346 std::optional<NnapiAccelerationTestParams> validation_params =
347 GetNnapiAccelerationTestParam(test_id);
348 if (!validation_params.has_value()) {
349 return;
350 }
351
352 // If we have multiple delegates applied, we would skip this check at the
353 // moment.
354 if (num_applied_delegates_ > 1) {
355 TFLITE_LOG(WARN) << "Skipping ExpectOpAcceleratedWithNnapi as "
356 << num_applied_delegates_
357 << " delegates have been successfully applied.";
358 return;
359 }
360 TFLITE_LOG(INFO) << "Validating acceleration";
361 const NnApi* nnapi = NnApiImplementation();
362 if (nnapi && nnapi->nnapi_exists &&
363 nnapi->android_sdk_version >=
364 validation_params.value().MinAndroidSdkVersion()) {
365 EXPECT_EQ(CountPartitionsDelegatedTo(interpreter_.get(), delegate_), 1)
366 << "Expecting operation to be accelerated but cannot find a partition "
367 "associated to the NNAPI delegate";
368 }
369 }
370
ValidateAcceleration()371 void SingleOpModel::ValidateAcceleration() {
372 if (GetForceUseNnapi()) {
373 ExpectOpAcceleratedWithNnapi(GetCurrentTestId());
374 }
375 }
376
CountOpsExecutedByCpuKernel()377 int SingleOpModel::CountOpsExecutedByCpuKernel() {
378 return CountPartitionsExecutedByCpuKernel(interpreter_.get());
379 }
380
~SingleOpModel()381 SingleOpModel::~SingleOpModel() { ValidateAcceleration(); }
382
AddBuiltinOp(BuiltinOperator type,BuiltinOptions builtin_options_type,const flatbuffers::Offset<void> & builtin_options,const std::vector<int32_t> & inputs,const std::vector<int32_t> & outputs)383 void MultiOpModel::AddBuiltinOp(
384 BuiltinOperator type, BuiltinOptions builtin_options_type,
385 const flatbuffers::Offset<void>& builtin_options,
386 const std::vector<int32_t>& inputs, const std::vector<int32_t>& outputs) {
387 opcodes_.push_back(CreateOperatorCode(builder_, type, 0, 0));
388 const int opcode_index = opcodes_.size() - 1;
389 operators_.push_back(CreateOperator(
390 builder_, opcode_index, builder_.CreateVector<int32_t>(inputs),
391 builder_.CreateVector<int32_t>(outputs), builtin_options_type,
392 builtin_options,
393 /*custom_options=*/0, CustomOptionsFormat_FLEXBUFFERS));
394 }
395
AddCustomOp(const string & name,const std::vector<uint8_t> & custom_option,const std::function<TfLiteRegistration * ()> & registration,const std::vector<int32_t> & inputs,const std::vector<int32_t> & outputs)396 void MultiOpModel::AddCustomOp(
397 const string& name, const std::vector<uint8_t>& custom_option,
398 const std::function<TfLiteRegistration*()>& registration,
399 const std::vector<int32_t>& inputs, const std::vector<int32_t>& outputs) {
400 custom_registrations_[name] = registration;
401 opcodes_.push_back(
402 CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
403 const int opcode_index = opcodes_.size() - 1;
404 operators_.push_back(CreateOperator(
405 builder_, opcode_index, builder_.CreateVector<int32_t>(inputs),
406 builder_.CreateVector<int32_t>(outputs), BuiltinOptions_NONE, 0,
407 builder_.CreateVector<uint8_t>(custom_option),
408 CustomOptionsFormat_FLEXBUFFERS));
409 }
410 } // namespace tflite
411