1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <gtest/gtest.h>
18
19 #include <algorithm>
20 #include <map>
21 #include <memory>
22 #include <set>
23 #include <string>
24 #include <utility>
25
26 #include "GeneratedTestUtils.h"
27 #include "TestHarness.h"
28 #include "TestNeuralNetworksWrapper.h"
29 #include "fuzzing/OperationManager.h"
30 #include "fuzzing/RandomGraphGenerator.h"
31 #include "fuzzing/RandomGraphGeneratorUtils.h"
32
33 #ifndef NNTEST_CTS
34 #include <android-base/properties.h>
35 #include <memunreachable/memunreachable.h>
36
37 #include <vector>
38
39 #include "HalInterfaces.h"
40 #include "Manager.h"
41 #include "SampleDriverFull.h"
42
43 using android::nn::sample_driver::SampleDriverFull;
44 using namespace android::nn::hal;
45
46 #endif
47
48 namespace android {
49 namespace nn {
50 namespace fuzzing_test {
51
52 using namespace test_helper;
53 using test_wrapper::Result;
54 constexpr char kRefDeviceName[] = "nnapi-reference";
55
56 #ifndef NNTEST_CTS
57 class TestDriverV1_2 : public SampleDriverFull {
58 public:
TestDriverV1_2()59 TestDriverV1_2() : SampleDriverFull(name, {.execTime = 0.9f, .powerUsage = 0.9f}) {}
60 static constexpr char name[] = "TestDriverV1_2";
61 };
62
63 // Like SampleDriverFull, but implementing 1.1
64 class TestDriverV1_1 : public V1_1::IDevice {
65 public:
TestDriverV1_1()66 TestDriverV1_1()
67 : mDriverV1_2(new SampleDriverFull(name, {.execTime = 0.8f, .powerUsage = 0.8f})) {}
68 static constexpr char name[] = "TestDriverV1_1";
getCapabilities_1_1(getCapabilities_1_1_cb _hidl_cb)69 Return<void> getCapabilities_1_1(getCapabilities_1_1_cb _hidl_cb) override {
70 return mDriverV1_2->getCapabilities_1_1(_hidl_cb);
71 }
getSupportedOperations_1_1(const V1_1::Model & model,getSupportedOperations_1_1_cb _hidl_cb)72 Return<void> getSupportedOperations_1_1(const V1_1::Model& model,
73 getSupportedOperations_1_1_cb _hidl_cb) override {
74 return mDriverV1_2->getSupportedOperations_1_1(model, _hidl_cb);
75 }
prepareModel_1_1(const V1_1::Model & model,ExecutionPreference preference,const sp<V1_0::IPreparedModelCallback> & actualCallback)76 Return<V1_0::ErrorStatus> prepareModel_1_1(
77 const V1_1::Model& model, ExecutionPreference preference,
78 const sp<V1_0::IPreparedModelCallback>& actualCallback) override {
79 return mDriverV1_2->prepareModel_1_1(model, preference, actualCallback);
80 }
getStatus()81 Return<DeviceStatus> getStatus() override { return mDriverV1_2->getStatus(); }
getCapabilities(getCapabilities_cb _hidl_cb)82 Return<void> getCapabilities(getCapabilities_cb _hidl_cb) override {
83 return mDriverV1_2->getCapabilities(_hidl_cb);
84 }
getSupportedOperations(const V1_0::Model & model,getSupportedOperations_cb _hidl_cb)85 Return<void> getSupportedOperations(const V1_0::Model& model,
86 getSupportedOperations_cb _hidl_cb) override {
87 return mDriverV1_2->getSupportedOperations(model, _hidl_cb);
88 }
prepareModel(const V1_0::Model & model,const sp<V1_0::IPreparedModelCallback> & actualCallback)89 Return<V1_0::ErrorStatus> prepareModel(
90 const V1_0::Model& model,
91 const sp<V1_0::IPreparedModelCallback>& actualCallback) override {
92 return mDriverV1_2->prepareModel(model, actualCallback);
93 }
94
95 private:
96 const sp<V1_2::IDevice> mDriverV1_2;
97 };
98
99 // Like SampleDriverFull, but implementing 1.0
100 class TestDriverV1_0 : public V1_0::IDevice {
101 public:
TestDriverV1_0()102 TestDriverV1_0()
103 : mDriverV1_2(new SampleDriverFull(name, {.execTime = 0.7f, .powerUsage = 0.7f})) {}
104 static constexpr char name[] = "TestDriverV1_0";
getCapabilities(getCapabilities_cb _hidl_cb)105 Return<void> getCapabilities(getCapabilities_cb _hidl_cb) override {
106 return mDriverV1_2->getCapabilities(_hidl_cb);
107 }
getSupportedOperations(const V1_0::Model & model,getSupportedOperations_cb _hidl_cb)108 Return<void> getSupportedOperations(const V1_0::Model& model,
109 getSupportedOperations_cb _hidl_cb) override {
110 return mDriverV1_2->getSupportedOperations(model, _hidl_cb);
111 }
prepareModel(const V1_0::Model & model,const sp<V1_0::IPreparedModelCallback> & actualCallback)112 Return<V1_0::ErrorStatus> prepareModel(
113 const V1_0::Model& model,
114 const sp<V1_0::IPreparedModelCallback>& actualCallback) override {
115 return mDriverV1_2->prepareModel(model, actualCallback);
116 }
getStatus()117 Return<DeviceStatus> getStatus() override { return mDriverV1_2->getStatus(); }
118
119 private:
120 const sp<V1_2::IDevice> mDriverV1_2;
121 };
122
123 template <class T_TestDriver>
makeTestDevice()124 std::shared_ptr<Device> makeTestDevice() {
125 return DeviceManager::forTest_makeDriverDevice(T_TestDriver::name, new T_TestDriver);
126 }
127
128 #endif
129
130 // NN API fuzzer logging setting comes from system property debug.nn.fuzzer.log and
131 // debug.nn.fuzzer.dumpspec.
132 // * setprop debug.nn.fuzzer.log 1 : enable logging.
133 // * setprop debug.nn.fuzzer.log 0 : silence logging.
134 // * setprop debug.nn.fuzzer.dumpspec 1 : dump the randomly generated graph to a spec file.
135 // * setprop debug.nn.fuzzer.dumpspec 0 : do not dump the graph.
136 //
137 // Logs and spec files are dumped to /data/local/tmp/${testname}.{log,mod.py},
138 // e.g. for test case TestRandomGraph/RandomGraphTest/Large/0,
139 // log : /data/local/tmp/TestRandomGraph_RandomGraphTest_Large_0.log
140 // spec: /data/local/tmp/TestRandomGraph_RandomGraphTest_Large_0.mod.py
141 //
142 class RandomGraphTest : public ::testing::TestWithParam<uint32_t> {
143 public:
SetUpTestCase()144 static void SetUpTestCase() {
145 #ifndef NNTEST_CTS
146 mEnableLog = ::android::base::GetProperty("debug.nn.fuzzer.log", "") == "1";
147 mDumpSpec = ::android::base::GetProperty("debug.nn.fuzzer.dumpspec", "") == "1";
148 mDetectMemoryLeak = ::android::base::GetProperty("debug.nn.fuzzer.detectleak", "") == "1";
149
150 mStandardDevices = DeviceManager::get()->forTest_getDevices();
151 mSyntheticDevices.push_back(makeTestDevice<TestDriverV1_2>());
152 mSyntheticDevices.push_back(makeTestDevice<TestDriverV1_1>());
153 mSyntheticDevices.push_back(makeTestDevice<TestDriverV1_0>());
154 #endif
155
156 // Get all the devices and device names.
157 mStandardDevicesFeatureLevel = __ANDROID_API_FUTURE__;
158 uint32_t numDevices = 0;
159 ASSERT_EQ(ANeuralNetworks_getDeviceCount(&numDevices), ANEURALNETWORKS_NO_ERROR);
160 for (uint32_t i = 0; i < numDevices; i++) {
161 ANeuralNetworksDevice* device = nullptr;
162 const char* name = nullptr;
163 int64_t featureLevel;
164 ASSERT_EQ(ANeuralNetworks_getDevice(i, &device), ANEURALNETWORKS_NO_ERROR);
165 ASSERT_EQ(ANeuralNetworksDevice_getName(device, &name), ANEURALNETWORKS_NO_ERROR);
166 ASSERT_EQ(ANeuralNetworksDevice_getFeatureLevel(device, &featureLevel),
167 ANEURALNETWORKS_NO_ERROR);
168 mDevices.emplace(name, device);
169 mStandardDevicesFeatureLevel = std::min(mStandardDevicesFeatureLevel, featureLevel);
170 }
171 }
172
173 protected:
SetUp()174 virtual void SetUp() override {
175 // Initialize logging.
176 const ::testing::TestInfo* const testInfo =
177 ::testing::UnitTest::GetInstance()->current_test_info();
178 mTestName = mTestName + testInfo->test_case_name() + "_" + testInfo->name();
179 std::replace(mTestName.begin(), mTestName.end(), '/', '_');
180 if (mEnableLog) NN_FUZZER_LOG_INIT("/data/local/tmp/" + mTestName + ".log");
181 }
182
TearDown()183 virtual void TearDown() override {
184 NN_FUZZER_LOG_CLOSE;
185 // Dump test results on failure for debugging.
186 if (::testing::Test::HasFailure() || mDumpSpec) {
187 dumpTestResults();
188 }
189 #ifndef NNTEST_CTS
190 if (mDetectMemoryLeak) {
191 ASSERT_TRUE(NoLeaks());
192 }
193 #endif
194 }
195
shouldSkipTest(int64_t featureLevel)196 bool shouldSkipTest(int64_t featureLevel) {
197 static const std::set<std::string> kDisabledTests = {
198 // In this test, the RGG produces a non-sensible graph with extreme large output
199 // gain and highly clamped output range.
200 // TODO: Currently quantized buffer values are uniformly distributed within
201 // [0, 255]. We should investigate on a better buffer value generation
202 // algorithm that represents the real-world cases.
203 "TestRandomGraph_SingleOperationTest_CONV_2D_V1_2_40",
204 "TestRandomGraph_SingleOperationTest_DEPTHWISE_CONV_2D_V1_0_32",
205 };
206 if (kDisabledTests.find(mTestName) != kDisabledTests.end()) return true;
207 for (const auto& op : mTestModel.main.operations) {
208 // Skip if testing BATCH_TO_SPACE_ND with batch dimension == 1.
209 if (op.type == TestOperationType::BATCH_TO_SPACE_ND &&
210 mTestModel.main.operands[op.inputs[0]].dimensions[0] == 1 &&
211 featureLevel <= __ANDROID_API_Q__) {
212 return true;
213 }
214 // L2_NORMALIZATION on axis of all zeros is undefined before R.
215 if (op.type == TestOperationType::L2_NORMALIZATION &&
216 featureLevel <= __ANDROID_API_Q__) {
217 return true;
218 }
219 // Skip the following operations for 1.2 and earlier devices.
220 if ((op.type == TestOperationType::ADD || op.type == TestOperationType::SUB ||
221 op.type == TestOperationType::MAXIMUM || op.type == TestOperationType::MINIMUM ||
222 op.type == TestOperationType::ROI_ALIGN) &&
223 mTestModel.main.operands[op.inputs[0]].type ==
224 TestOperandType::TENSOR_QUANT8_ASYMM &&
225 featureLevel <= __ANDROID_API_Q__) {
226 return true;
227 }
228 }
229 return false;
230 }
231
232 // Compute the golden output results of the test model on nnapi-reference. If possible, the
233 // golden results will be computed from an equivalent float32 model to avoid bias avoid bias
234 // from quantized CPU implementation.
computeGoldenResults()235 void computeGoldenResults() {
236 SCOPED_TRACE("computeGoldenResults");
237
238 // Convert the test model to an equivalent float32 model if possible.
239 auto fpModel = convertToFloat32Model(mTestModel);
240 const TestModel& goldenModel = fpModel.has_value() ? fpModel.value() : mTestModel;
241
242 // Create model.
243 generated_tests::GeneratedModel model;
244 generated_tests::createModel(goldenModel, &model);
245 ASSERT_TRUE(model.isValid());
246 ASSERT_EQ(model.finish(), Result::NO_ERROR);
247
248 // Create compilation for nnapi-reference.
249 ASSERT_TRUE(mDevices.find(kRefDeviceName) != mDevices.end());
250 const auto refDevice = mDevices[kRefDeviceName];
251 auto [result, compilation] = test_wrapper::Compilation::createForDevice(&model, refDevice);
252 ASSERT_EQ(result, Result::NO_ERROR);
253 ASSERT_EQ(compilation.finish(), Result::NO_ERROR);
254
255 // Create request.
256 test_wrapper::Execution execution(&compilation);
257 std::vector<TestBuffer> outputs;
258 generated_tests::createRequest(goldenModel, &execution, &outputs);
259
260 // Compute result.
261 ASSERT_EQ(execution.compute(), Result::NO_ERROR);
262
263 if (fpModel.has_value()) {
264 // Quantize the execution results as golden values.
265 setExpectedOutputsFromFloat32Results(outputs, &mTestModel);
266 } else {
267 for (uint32_t i = 0; i < outputs.size(); i++) {
268 auto outputIndex = mTestModel.main.outputIndexes[i];
269 mTestModel.main.operands[outputIndex].data = outputs[i];
270 }
271 }
272 }
273
274 // Compile and execute the generated graph on a device selected by name.
computeAndVerifyResultsForDevice(const test_wrapper::Model * model,uint32_t numOps,const std::string & name)275 void computeAndVerifyResultsForDevice(const test_wrapper::Model* model, uint32_t numOps,
276 const std::string& name) {
277 SCOPED_TRACE("Device: " + name);
278 std::cout << "[ ] - RUN: " << name << "\n";
279 ASSERT_TRUE(mDevices.find(name) != mDevices.end());
280 const auto device = mDevices[name];
281
282 // Check if the device fully supports the graph.
283 constexpr int kMaxNumberOperations = 1000;
284 ASSERT_TRUE(numOps <= kMaxNumberOperations);
285 bool supported[kMaxNumberOperations] = {false};
286 ASSERT_EQ(ANeuralNetworksModel_getSupportedOperationsForDevices(model->getHandle(), &device,
287 1, supported),
288 ANEURALNETWORKS_NO_ERROR);
289 if (!std::all_of(supported, supported + numOps, [](bool v) { return v; })) {
290 std::cout << "[ ] SKIP: " << name << " does not support the graph.\n";
291 return;
292 }
293
294 // Since this test is introduced in Android Q, we only check the accuracy of output results
295 // if the device has feature level >= Q (API level 29). For pre-Q devices, we allow
296 // them to produce less accurate results, but must not hang or crash.
297 int64_t featureLevel;
298 ASSERT_EQ(ANeuralNetworksDevice_getFeatureLevel(device, &featureLevel),
299 ANEURALNETWORKS_NO_ERROR);
300 if (shouldSkipTest(featureLevel)) return;
301
302 // Create compilation for device.
303 auto [result, compilation] = test_wrapper::Compilation::createForDevice(model, device);
304 ASSERT_EQ(result, Result::NO_ERROR);
305 Result compileReturn = compilation.finish();
306 // Even if the model is fully supported, the compilation may still fail, e.g. each operation
307 // is supported, but model is too big (too many operations and/or too-large constants) for
308 // device.
309 if (compileReturn == Result::OP_FAILED) {
310 std::cout << "[ ] SKIP: " << name << " failed at compilation step.\n";
311 return;
312 }
313 ASSERT_EQ(compileReturn, Result::NO_ERROR);
314
315 // Create request.
316 test_wrapper::Execution execution(&compilation);
317 std::vector<TestBuffer> outputs;
318 generated_tests::createRequest(mTestModel, &execution, &outputs);
319
320 // Compute result.
321 Result executeReturn = execution.compute();
322 // Even if the model is fully supported and the compilation succeeds, the execution may
323 // still fail, e.g. there may be operand shapes that are unknown until execution time, and
324 // at execution time turn out to be too big.
325 if (executeReturn == Result::OP_FAILED) {
326 std::cout << "[ ] SKIP: " << name << " failed at execution step.\n";
327 return;
328 }
329 ASSERT_EQ(executeReturn, Result::NO_ERROR);
330
331 if (featureLevel >= __ANDROID_API_Q__) {
332 checkResults(mTestModel, outputs, mCriteria);
333 mResults.emplace_back(name, std::move(outputs));
334 }
335 }
336
337 // Compile and execute the generated graph normally (i.e., allow runtime to
338 // distribute across devices).
computeAndVerifyResults(const std::string & name,const test_wrapper::Model * model,bool shouldCheckResults)339 void computeAndVerifyResults(const std::string& name, const test_wrapper::Model* model,
340 bool shouldCheckResults) {
341 // Because we're not using the introspection/control API, the CpuDevice
342 // is available as a fallback, and hence we assume that compilation and
343 // execution will succeed.
344 SCOPED_TRACE(name);
345 std::cout << "[ ] - RUN: " << name << "\n";
346
347 // Create compilation.
348 test_wrapper::Compilation compilation(model);
349 ASSERT_EQ(compilation.finish(), Result::NO_ERROR);
350
351 // Create request.
352 test_wrapper::Execution execution(&compilation);
353 std::vector<TestBuffer> outputs;
354 generated_tests::createRequest(mTestModel, &execution, &outputs);
355
356 // Compute and verify result.
357 ASSERT_EQ(execution.compute(), Result::NO_ERROR);
358 if (shouldCheckResults) {
359 checkResults(mTestModel, outputs, mCriteria);
360 mResults.emplace_back(name, std::move(outputs));
361 }
362 }
363
364 // Main test entrance.
testRandomGraph(uint32_t numOperations,uint32_t dimensionRange)365 void testRandomGraph(uint32_t numOperations, uint32_t dimensionRange) {
366 // Generate a random graph.
367 RandomGraph graph;
368 ASSERT_TRUE(graph.generate(kSeed, numOperations, dimensionRange));
369
370 // Create a model from the random graph.
371 mTestModel = graph.createTestModel();
372
373 generated_tests::GeneratedModel model;
374 generated_tests::createModel(mTestModel, &model);
375 ASSERT_TRUE(model.isValid());
376 ASSERT_EQ(model.finish(), Result::NO_ERROR);
377
378 // Compute reference results.
379 computeGoldenResults();
380
381 // Compute on each available device.
382 for (auto& pair : mDevices) {
383 computeAndVerifyResultsForDevice(&model, numOperations, pair.first);
384 }
385
386 if (numOperations > 1) {
387 if (!shouldSkipTest(mStandardDevicesFeatureLevel)) {
388 // Compute normally (i.e., allow runtime to distribute across devices).
389 computeAndVerifyResults("Compute normally", &model,
390 mStandardDevicesFeatureLevel >= __ANDROID_API_Q__);
391 }
392
393 #ifndef NNTEST_CTS
394 {
395 // Stress partitioner by allowing runtime to distribute across
396 // three synthetic devices. The synthetic devices use the
397 // CpuExecutor for execution, so we always check results, even
398 // though some are of feature level < __ANDROID_API_Q__: In this
399 // case, we don't take feature level as an indication of
400 // reliability, as we do with real devices.
401 DeviceManager::get()->forTest_setDevices(mSyntheticDevices);
402 computeAndVerifyResults("Compute across synthetic devices", &model, true);
403 DeviceManager::get()->forTest_setDevices(mStandardDevices);
404 }
405 #endif
406 }
407 }
408
dumpTestResults()409 void dumpTestResults() {
410 std::ofstream os("/data/local/tmp/" + mTestName + ".mod.py");
411 ASSERT_TRUE(os.is_open());
412 os << "# Generated from " << mTestName << ". Do not edit.\n\n";
413 SpecDumper dumper(mTestModel, os);
414 dumper.dumpTestModel();
415 for (const auto& [name, results] : mResults) {
416 dumper.dumpResults(name, results);
417 }
418 }
419
420 enum GraphSize : uint32_t { SINGLE = 1, SMALL = 5, LARGE = 40 };
421 enum DimensionRange : uint32_t { NARROW = 10, WIDE = 1000 };
422
423 static bool mEnableLog;
424 static bool mDumpSpec;
425 static bool mDetectMemoryLeak;
426 static std::map<std::string, ANeuralNetworksDevice*> mDevices;
427
428 const uint32_t kSeed = GetParam();
429 std::string mTestName;
430 TestModel mTestModel;
431 AccuracyCriteria mCriteria;
432
433 // A vector of {name, output_results}.
434 std::vector<std::pair<std::string, std::vector<TestBuffer>>> mResults;
435
436 static int64_t mStandardDevicesFeatureLevel; // minimum across all devices
437 #ifndef NNTEST_CTS
438 static std::vector<std::shared_ptr<Device>> mStandardDevices;
439 static std::vector<std::shared_ptr<Device>> mSyntheticDevices;
440 #endif
441 };
442
443 bool RandomGraphTest::mEnableLog = false;
444 bool RandomGraphTest::mDumpSpec = false;
445 bool RandomGraphTest::mDetectMemoryLeak = false;
446 std::map<std::string, ANeuralNetworksDevice*> RandomGraphTest::mDevices;
447
448 int64_t RandomGraphTest::mStandardDevicesFeatureLevel;
449 #ifndef NNTEST_CTS
450 std::vector<std::shared_ptr<Device>> RandomGraphTest::mStandardDevices;
451 std::vector<std::shared_ptr<Device>> RandomGraphTest::mSyntheticDevices;
452 #endif
453
454 // Single-op graph with dimensions in range [1, 1000].
455 class SingleOperationTest : public RandomGraphTest {};
456 #define TEST_SINGLE_OPERATION(operation, halVersion, criteria) \
457 TEST_P(SingleOperationTest, operation##_##halVersion) { \
458 OperationFilter filter = {.opcodes = {TestOperationType::operation}, \
459 .versions = {TestHalVersion::halVersion}}; \
460 OperationManager::get()->applyFilter(filter); \
461 mCriteria = (criteria); \
462 testRandomGraph(GraphSize::SINGLE, DimensionRange::WIDE); \
463 }
464
465 // TODO: Adjust the accuracy criteria based on testing.
466 // We define three sets of accuracy criteria for single-operation tests.
467
468 // This is for operations that only copy buffers around without any computation on buffer values.
469 // Most of these operations fall into categories of reshape or selection, e.g. RESHAPE, GATHER.
470 // Additionally, operations with only logical or comparison arithmetic also use this criteria, e.g.
471 // EQUAL, ARGMAX, TOPK_V2.
472 const AccuracyCriteria kStrictCriteria = {
473 .float32 = {.bias = 1e-7f, .mse = 1e-10f, .atol = 1e-6f, .rtol = 1e-6f},
474 .float16 = {.bias = 1e-4f, .mse = 1e-8f, .atol = 1e-3f, .rtol = 1e-3f},
475 .int32 = {.atol = 1},
476 .quant8Asymm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
477 .quant8AsymmSigned = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
478 .quant8Symm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
479 .quant16Asymm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
480 .quant16Symm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
481 };
482
483 // This is for operations that only do simple and single computation on buffer values, such as
484 // addition, multiplication, or requantization. Most of these operations fall into categories of
485 // broadcast or elementwise, e.g ADD, FLOOR.
486 const AccuracyCriteria kMediumCriteria = {
487 .float32 = {.bias = 1e-6f, .mse = 1e-8f, .atol = 1e-5f, .rtol = 1e-5f},
488 .float16 = {.bias = 1e-3f, .mse = 1e-5f, .atol = 1e-2f, .rtol = 1e-2f},
489 .int32 = {.atol = 1},
490 .quant8Asymm = {.bias = 1.2, .mse = 1.2, .atol = 2},
491 .quant8AsymmSigned = {.bias = 1.2, .mse = 1.2, .atol = 2},
492 .quant8Symm = {.bias = 1.2, .mse = 1.2, .atol = 2},
493 .quant16Asymm = {.bias = 1.2, .mse = 1.2, .atol = 2},
494 .quant16Symm = {.bias = 1.2, .mse = 1.2, .atol = 2},
495 };
496
497 // This is for operations that involve sophisticated computations on buffer values, either a single
498 // but complex transformation, e.g. LOGISTIC, or multiple transformations with accumulated errors,
499 // e.g. L2_NORMALIZATION, REDUCE_*.
500 const AccuracyCriteria kRelaxedCriteria = {
501 .float32 = {.bias = 3e-5f, .mse = 1e-6f, .atol = 1e-3f, .rtol = 1e-3f},
502 .float16 = {.bias = 5e-3f, .mse = 1e-3f, .atol = 1.0f, .rtol = 1.0f},
503 .int32 = {.atol = 1},
504 .quant8Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
505 .quant8AsymmSigned = {.bias = 1.5, .mse = 1.5, .atol = 10},
506 .quant8Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
507 .quant16Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
508 .quant16Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
509 };
510
511 // This is for convolution operations with potentially large kernel size.
512 const AccuracyCriteria kConvCriteria = {
513 .float32 = {.bias = 4e-4f, .mse = 1e-5f, .atol = 2e-2f, .rtol = 2e-2f},
514 .float16 = {.bias = 5e-2f, .mse = 1e-2f, .atol = 1.0f, .rtol = 1.0f},
515 .int32 = {.atol = 1},
516 .quant8Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
517 .quant8AsymmSigned = {.bias = 1.5, .mse = 1.5, .atol = 10},
518 .quant8Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
519 .quant16Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
520 .quant16Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
521 };
522
523 /*-- NNAPI 1.0 Operations ---------------------------------------------------*/
524
525 // TODO: The following 1.0 operation signatures are currently not defined:
526 // - ANEURALNETWORKS_LSH_PROJECTION
527 // - ANEURALNETWORKS_LSTM
528 // - ANEURALNETWORKS_RNN
529 // - ANEURALNETWORKS_SVDF
530
531 TEST_SINGLE_OPERATION(ADD, V1_0, kMediumCriteria);
532 TEST_SINGLE_OPERATION(MUL, V1_0, kMediumCriteria);
533 TEST_SINGLE_OPERATION(FLOOR, V1_0, kMediumCriteria);
534 TEST_SINGLE_OPERATION(LOGISTIC, V1_0, kRelaxedCriteria);
535 TEST_SINGLE_OPERATION(RELU, V1_0, kMediumCriteria);
536 TEST_SINGLE_OPERATION(RELU1, V1_0, kMediumCriteria);
537 TEST_SINGLE_OPERATION(RELU6, V1_0, kMediumCriteria);
538 TEST_SINGLE_OPERATION(TANH, V1_0, kRelaxedCriteria);
539 TEST_SINGLE_OPERATION(SOFTMAX, V1_0, kRelaxedCriteria);
540 TEST_SINGLE_OPERATION(L2_NORMALIZATION, V1_0, kRelaxedCriteria);
541 TEST_SINGLE_OPERATION(LOCAL_RESPONSE_NORMALIZATION, V1_0, kRelaxedCriteria);
542 TEST_SINGLE_OPERATION(AVERAGE_POOL_2D, V1_0, kRelaxedCriteria);
543 TEST_SINGLE_OPERATION(L2_POOL_2D, V1_0, kRelaxedCriteria);
544 TEST_SINGLE_OPERATION(MAX_POOL_2D, V1_0, kRelaxedCriteria);
545 TEST_SINGLE_OPERATION(CONV_2D, V1_0, kConvCriteria);
546 TEST_SINGLE_OPERATION(DEPTHWISE_CONV_2D, V1_0, kConvCriteria);
547 TEST_SINGLE_OPERATION(CONCATENATION, V1_0, kMediumCriteria);
548 TEST_SINGLE_OPERATION(RESIZE_BILINEAR, V1_0, kRelaxedCriteria);
549 TEST_SINGLE_OPERATION(DEPTH_TO_SPACE, V1_0, kStrictCriteria);
550 TEST_SINGLE_OPERATION(SPACE_TO_DEPTH, V1_0, kStrictCriteria);
551 TEST_SINGLE_OPERATION(EMBEDDING_LOOKUP, V1_0, kStrictCriteria);
552 TEST_SINGLE_OPERATION(HASHTABLE_LOOKUP, V1_0, kStrictCriteria);
553 TEST_SINGLE_OPERATION(FULLY_CONNECTED, V1_0, kRelaxedCriteria);
554 TEST_SINGLE_OPERATION(RESHAPE, V1_0, kStrictCriteria);
555 TEST_SINGLE_OPERATION(DEQUANTIZE, V1_0, kMediumCriteria);
556
557 /*-- NNAPI 1.1 Operations ---------------------------------------------------*/
558
559 TEST_SINGLE_OPERATION(SUB, V1_1, kMediumCriteria);
560 TEST_SINGLE_OPERATION(DIV, V1_1, kRelaxedCriteria);
561 TEST_SINGLE_OPERATION(BATCH_TO_SPACE_ND, V1_1, kStrictCriteria);
562 TEST_SINGLE_OPERATION(SPACE_TO_BATCH_ND, V1_1, kStrictCriteria);
563 TEST_SINGLE_OPERATION(MEAN, V1_1, kRelaxedCriteria);
564 TEST_SINGLE_OPERATION(PAD, V1_1, kStrictCriteria);
565 TEST_SINGLE_OPERATION(TRANSPOSE, V1_1, kStrictCriteria);
566 TEST_SINGLE_OPERATION(SQUEEZE, V1_1, kStrictCriteria);
567 TEST_SINGLE_OPERATION(STRIDED_SLICE, V1_1, kStrictCriteria);
568
569 /*-- NNAPI 1.0 and 1.1 Operations with Extended Behavior in 1.2 -------------*/
570
571 TEST_SINGLE_OPERATION(ADD, V1_2, kMediumCriteria);
572 TEST_SINGLE_OPERATION(MUL, V1_2, kMediumCriteria);
573 TEST_SINGLE_OPERATION(SUB, V1_2, kMediumCriteria);
574 TEST_SINGLE_OPERATION(DIV, V1_2, kRelaxedCriteria);
575 TEST_SINGLE_OPERATION(FLOOR, V1_2, kMediumCriteria);
576 TEST_SINGLE_OPERATION(LOGISTIC, V1_2, kRelaxedCriteria);
577 TEST_SINGLE_OPERATION(RELU, V1_2, kMediumCriteria);
578 TEST_SINGLE_OPERATION(RELU1, V1_2, kMediumCriteria);
579 TEST_SINGLE_OPERATION(RELU6, V1_2, kMediumCriteria);
580 TEST_SINGLE_OPERATION(TANH, V1_2, kRelaxedCriteria);
581 TEST_SINGLE_OPERATION(CONCATENATION, V1_2, kMediumCriteria);
582 TEST_SINGLE_OPERATION(DEPTH_TO_SPACE, V1_2, kStrictCriteria);
583 TEST_SINGLE_OPERATION(SPACE_TO_DEPTH, V1_2, kStrictCriteria);
584 TEST_SINGLE_OPERATION(BATCH_TO_SPACE_ND, V1_2, kStrictCriteria);
585 TEST_SINGLE_OPERATION(SPACE_TO_BATCH_ND, V1_2, kStrictCriteria);
586 TEST_SINGLE_OPERATION(FULLY_CONNECTED, V1_2, kRelaxedCriteria);
587 TEST_SINGLE_OPERATION(RESHAPE, V1_2, kStrictCriteria);
588 TEST_SINGLE_OPERATION(MEAN, V1_2, kRelaxedCriteria);
589 TEST_SINGLE_OPERATION(PAD, V1_2, kStrictCriteria);
590 TEST_SINGLE_OPERATION(TRANSPOSE, V1_2, kStrictCriteria);
591 TEST_SINGLE_OPERATION(CONV_2D, V1_2, kConvCriteria);
592 TEST_SINGLE_OPERATION(DEPTHWISE_CONV_2D, V1_2, kConvCriteria);
593 TEST_SINGLE_OPERATION(AVERAGE_POOL_2D, V1_2, kRelaxedCriteria);
594 TEST_SINGLE_OPERATION(L2_POOL_2D, V1_2, kRelaxedCriteria);
595 TEST_SINGLE_OPERATION(MAX_POOL_2D, V1_2, kRelaxedCriteria);
596 TEST_SINGLE_OPERATION(RESIZE_BILINEAR, V1_2, kRelaxedCriteria);
597 TEST_SINGLE_OPERATION(SOFTMAX, V1_2, kRelaxedCriteria);
598 TEST_SINGLE_OPERATION(L2_NORMALIZATION, V1_2, kRelaxedCriteria);
599 TEST_SINGLE_OPERATION(LOCAL_RESPONSE_NORMALIZATION, V1_2, kRelaxedCriteria);
600 TEST_SINGLE_OPERATION(DEQUANTIZE, V1_2, kMediumCriteria);
601 TEST_SINGLE_OPERATION(SQUEEZE, V1_2, kStrictCriteria);
602 TEST_SINGLE_OPERATION(STRIDED_SLICE, V1_2, kStrictCriteria);
603 TEST_SINGLE_OPERATION(EMBEDDING_LOOKUP, V1_2, kStrictCriteria);
604
605 /*-- NNAPI 1.2 Operations ---------------------------------------------------*/
606
607 // TODO: The following 1.2 operation signatures are currently not defined:
608 // - ANEURALNETWORKS_AXIS_ALIGNED_BBOX_TRANSFORM
609 // - ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_LSTM
610 // - ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_RNN
611 // - ANEURALNETWORKS_BOX_WITH_NMS_LIMIT
612 // - ANEURALNETWORKS_DETECTION_POSTPROCESSING
613 // - ANEURALNETWORKS_GENERATE_PROPOSALS
614 // - ANEURALNETWORKS_QUANTIZED_16BIT_LSTM
615 // - ANEURALNETWORKS_RANDOM_MULTINOMIAL
616 // - ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_LSTM
617 // - ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_RNN
618
619 TEST_SINGLE_OPERATION(ABS, V1_2, kMediumCriteria);
620 TEST_SINGLE_OPERATION(EXP, V1_2, kRelaxedCriteria);
621 TEST_SINGLE_OPERATION(LOG, V1_2, kRelaxedCriteria);
622 TEST_SINGLE_OPERATION(NEG, V1_2, kMediumCriteria);
623 TEST_SINGLE_OPERATION(RSQRT, V1_2, kRelaxedCriteria);
624 TEST_SINGLE_OPERATION(SIN, V1_2, kRelaxedCriteria);
625 TEST_SINGLE_OPERATION(SQRT, V1_2, kRelaxedCriteria);
626 TEST_SINGLE_OPERATION(ARGMAX, V1_2, kStrictCriteria);
627 TEST_SINGLE_OPERATION(ARGMIN, V1_2, kStrictCriteria);
628 TEST_SINGLE_OPERATION(EQUAL, V1_2, kStrictCriteria);
629 TEST_SINGLE_OPERATION(GREATER, V1_2, kStrictCriteria);
630 TEST_SINGLE_OPERATION(GREATER_EQUAL, V1_2, kStrictCriteria);
631 TEST_SINGLE_OPERATION(LESS, V1_2, kStrictCriteria);
632 TEST_SINGLE_OPERATION(LESS_EQUAL, V1_2, kStrictCriteria);
633 TEST_SINGLE_OPERATION(LOGICAL_AND, V1_2, kStrictCriteria);
634 TEST_SINGLE_OPERATION(LOGICAL_NOT, V1_2, kStrictCriteria);
635 TEST_SINGLE_OPERATION(LOGICAL_OR, V1_2, kStrictCriteria);
636 TEST_SINGLE_OPERATION(NOT_EQUAL, V1_2, kStrictCriteria);
637 TEST_SINGLE_OPERATION(MAXIMUM, V1_2, kMediumCriteria);
638 TEST_SINGLE_OPERATION(MINIMUM, V1_2, kMediumCriteria);
639 TEST_SINGLE_OPERATION(POW, V1_2, kRelaxedCriteria);
640 TEST_SINGLE_OPERATION(PRELU, V1_2, kMediumCriteria);
641 TEST_SINGLE_OPERATION(REDUCE_ALL, V1_2, kRelaxedCriteria);
642 TEST_SINGLE_OPERATION(REDUCE_ANY, V1_2, kRelaxedCriteria);
643 TEST_SINGLE_OPERATION(REDUCE_MAX, V1_2, kRelaxedCriteria);
644 TEST_SINGLE_OPERATION(REDUCE_MIN, V1_2, kRelaxedCriteria);
645 TEST_SINGLE_OPERATION(REDUCE_PROD, V1_2, kRelaxedCriteria);
646 TEST_SINGLE_OPERATION(REDUCE_SUM, V1_2, kRelaxedCriteria);
647 TEST_SINGLE_OPERATION(CHANNEL_SHUFFLE, V1_2, kStrictCriteria);
648 TEST_SINGLE_OPERATION(INSTANCE_NORMALIZATION, V1_2, kRelaxedCriteria);
649 TEST_SINGLE_OPERATION(LOG_SOFTMAX, V1_2, kRelaxedCriteria);
650 TEST_SINGLE_OPERATION(GROUPED_CONV_2D, V1_2, kConvCriteria);
651 TEST_SINGLE_OPERATION(TRANSPOSE_CONV_2D, V1_2, kConvCriteria);
652 TEST_SINGLE_OPERATION(RESIZE_NEAREST_NEIGHBOR, V1_2, kRelaxedCriteria);
653 TEST_SINGLE_OPERATION(PAD_V2, V1_2, kStrictCriteria);
654 TEST_SINGLE_OPERATION(QUANTIZE, V1_2, kMediumCriteria);
655 TEST_SINGLE_OPERATION(CAST, V1_2, kMediumCriteria);
656 TEST_SINGLE_OPERATION(EXPAND_DIMS, V1_2, kStrictCriteria);
657 TEST_SINGLE_OPERATION(TILE, V1_2, kStrictCriteria);
658 TEST_SINGLE_OPERATION(GATHER, V1_2, kStrictCriteria);
659 TEST_SINGLE_OPERATION(SELECT, V1_2, kStrictCriteria);
660 TEST_SINGLE_OPERATION(TOPK_V2, V1_2, kStrictCriteria);
661 TEST_SINGLE_OPERATION(SLICE, V1_2, kStrictCriteria);
662 TEST_SINGLE_OPERATION(SPLIT, V1_2, kMediumCriteria);
663 TEST_SINGLE_OPERATION(ROI_ALIGN, V1_2, kRelaxedCriteria);
664 TEST_SINGLE_OPERATION(ROI_POOLING, V1_2, kRelaxedCriteria);
665 TEST_SINGLE_OPERATION(HEATMAP_MAX_KEYPOINT, V1_2, kRelaxedCriteria);
666
667 /*-- NNAPI 1.0, 1.1, and 1.2 Operations with Extended Behavior in 1.3 -------------*/
668
669 TEST_SINGLE_OPERATION(ADD, V1_3, kMediumCriteria);
670 TEST_SINGLE_OPERATION(AVERAGE_POOL_2D, V1_3, kRelaxedCriteria);
671 TEST_SINGLE_OPERATION(CONCATENATION, V1_3, kMediumCriteria);
672 TEST_SINGLE_OPERATION(CONV_2D, V1_3, kConvCriteria);
673 TEST_SINGLE_OPERATION(DEPTHWISE_CONV_2D, V1_3, kConvCriteria);
674 TEST_SINGLE_OPERATION(DEPTH_TO_SPACE, V1_3, kStrictCriteria);
675 TEST_SINGLE_OPERATION(DEQUANTIZE, V1_3, kMediumCriteria);
676 TEST_SINGLE_OPERATION(EMBEDDING_LOOKUP, V1_3, kStrictCriteria);
677 TEST_SINGLE_OPERATION(FULLY_CONNECTED, V1_3, kRelaxedCriteria);
678 TEST_SINGLE_OPERATION(L2_NORMALIZATION, V1_3, kRelaxedCriteria);
679 TEST_SINGLE_OPERATION(LOGISTIC, V1_3, kRelaxedCriteria);
680 TEST_SINGLE_OPERATION(MAX_POOL_2D, V1_3, kRelaxedCriteria);
681 TEST_SINGLE_OPERATION(MUL, V1_3, kMediumCriteria);
682 TEST_SINGLE_OPERATION(RELU, V1_3, kMediumCriteria);
683 TEST_SINGLE_OPERATION(RELU1, V1_3, kMediumCriteria);
684 TEST_SINGLE_OPERATION(RELU6, V1_3, kMediumCriteria);
685 TEST_SINGLE_OPERATION(RESHAPE, V1_3, kStrictCriteria);
686 TEST_SINGLE_OPERATION(RESIZE_BILINEAR, V1_3, kRelaxedCriteria);
687 TEST_SINGLE_OPERATION(SOFTMAX, V1_3, kRelaxedCriteria);
688 TEST_SINGLE_OPERATION(SPACE_TO_DEPTH, V1_3, kStrictCriteria);
689 TEST_SINGLE_OPERATION(TANH, V1_3, kRelaxedCriteria);
690 TEST_SINGLE_OPERATION(BATCH_TO_SPACE_ND, V1_3, kStrictCriteria);
691 TEST_SINGLE_OPERATION(DIV, V1_3, kMediumCriteria);
692 TEST_SINGLE_OPERATION(MEAN, V1_3, kRelaxedCriteria);
693 TEST_SINGLE_OPERATION(PAD, V1_3, kStrictCriteria);
694 TEST_SINGLE_OPERATION(SPACE_TO_BATCH_ND, V1_3, kStrictCriteria);
695 TEST_SINGLE_OPERATION(SQUEEZE, V1_3, kStrictCriteria);
696 TEST_SINGLE_OPERATION(STRIDED_SLICE, V1_3, kStrictCriteria);
697 TEST_SINGLE_OPERATION(SUB, V1_3, kMediumCriteria);
698 TEST_SINGLE_OPERATION(TRANSPOSE, V1_3, kStrictCriteria);
699 TEST_SINGLE_OPERATION(ABS, V1_3, kMediumCriteria);
700 TEST_SINGLE_OPERATION(ARGMAX, V1_3, kStrictCriteria);
701 TEST_SINGLE_OPERATION(ARGMIN, V1_3, kStrictCriteria);
702 TEST_SINGLE_OPERATION(CAST, V1_3, kMediumCriteria);
703 TEST_SINGLE_OPERATION(CHANNEL_SHUFFLE, V1_3, kStrictCriteria);
704 TEST_SINGLE_OPERATION(EQUAL, V1_3, kStrictCriteria);
705 TEST_SINGLE_OPERATION(EXPAND_DIMS, V1_3, kStrictCriteria);
706 TEST_SINGLE_OPERATION(GATHER, V1_3, kStrictCriteria);
707 TEST_SINGLE_OPERATION(GREATER, V1_3, kStrictCriteria);
708 TEST_SINGLE_OPERATION(GREATER_EQUAL, V1_3, kStrictCriteria);
709 TEST_SINGLE_OPERATION(GROUPED_CONV_2D, V1_3, kConvCriteria);
710 TEST_SINGLE_OPERATION(HEATMAP_MAX_KEYPOINT, V1_3, kRelaxedCriteria);
711 TEST_SINGLE_OPERATION(LESS, V1_3, kStrictCriteria);
712 TEST_SINGLE_OPERATION(LESS_EQUAL, V1_3, kStrictCriteria);
713 TEST_SINGLE_OPERATION(MAXIMUM, V1_3, kMediumCriteria);
714 TEST_SINGLE_OPERATION(MINIMUM, V1_3, kMediumCriteria);
715 TEST_SINGLE_OPERATION(NOT_EQUAL, V1_3, kStrictCriteria);
716 TEST_SINGLE_OPERATION(PAD_V2, V1_3, kStrictCriteria);
717 TEST_SINGLE_OPERATION(PRELU, V1_3, kMediumCriteria);
718 TEST_SINGLE_OPERATION(QUANTIZE, V1_3, kMediumCriteria);
719 TEST_SINGLE_OPERATION(REDUCE_MAX, V1_3, kRelaxedCriteria);
720 TEST_SINGLE_OPERATION(REDUCE_MIN, V1_3, kRelaxedCriteria);
721 TEST_SINGLE_OPERATION(ROI_ALIGN, V1_3, kRelaxedCriteria);
722 TEST_SINGLE_OPERATION(ROI_POOLING, V1_3, kRelaxedCriteria);
723 TEST_SINGLE_OPERATION(SELECT, V1_3, kStrictCriteria);
724 TEST_SINGLE_OPERATION(SLICE, V1_3, kStrictCriteria);
725 TEST_SINGLE_OPERATION(SPLIT, V1_3, kMediumCriteria);
726 TEST_SINGLE_OPERATION(TILE, V1_3, kStrictCriteria);
727 TEST_SINGLE_OPERATION(TOPK_V2, V1_3, kStrictCriteria);
728 TEST_SINGLE_OPERATION(TRANSPOSE_CONV_2D, V1_3, kConvCriteria);
729 TEST_SINGLE_OPERATION(RESIZE_NEAREST_NEIGHBOR, V1_3, kRelaxedCriteria);
730
731 /*-- NNAPI 1.3 Operations ---------------------------------------------------*/
732
733 // TODO: The following 1.3 operation signatures are currently not defined:
734 // - ANEURALNETWORKS_QUANTIZED_LSTM
735 // - ANEURALNETWORKS_IF
736 // - ANEURALNETWORKS_WHILE
737
738 TEST_SINGLE_OPERATION(ELU, V1_3, kMediumCriteria);
739 TEST_SINGLE_OPERATION(HARD_SWISH, V1_3, kMediumCriteria);
740 TEST_SINGLE_OPERATION(FILL, V1_3, kStrictCriteria);
741 TEST_SINGLE_OPERATION(RANK, V1_3, kStrictCriteria);
742
743 const AccuracyCriteria kSmallGraphCriteria = {
744 .float32 = {.bias = 4e-4f, .mse = 1e-5f, .atol = 1e-2f, .rtol = 1e-2f},
745 .float16 = {.bias = 5e-2f, .mse = 1e-2f, .atol = 1.0f, .rtol = 1.0f},
746 .int32 = {.atol = 1},
747 .quant8Asymm = {.bias = 2, .mse = 2, .atol = 12},
748 .quant8AsymmSigned = {.bias = 2, .mse = 2, .atol = 12},
749 .quant8Symm = {.bias = 2, .mse = 2, .atol = 12},
750 .quant16Asymm = {.bias = 2, .mse = 2, .atol = 12},
751 .quant16Symm = {.bias = 2, .mse = 2, .atol = 12},
752 };
753
754 const AccuracyCriteria kLargeGraphCriteria = {
755 .float32 = {.bias = 1e-2f, .mse = 1e-4f, .atol = 1e-1f, .rtol = 1e-1f},
756 .float16 = {.bias = 1e-1f, .mse = 5e-2f, .atol = 1.0f, .rtol = 1.0f},
757 .int32 = {.atol = 1},
758 .quant8Asymm = {.bias = 2, .mse = 2, .atol = 12},
759 .quant8AsymmSigned = {.bias = 2, .mse = 2, .atol = 12},
760 .quant8Symm = {.bias = 2, .mse = 2, .atol = 12},
761 .quant16Asymm = {.bias = 2, .mse = 2, .atol = 12},
762 .quant16Symm = {.bias = 2, .mse = 2, .atol = 12},
763 };
764
765 // Due to the limitation of the random graph generator, graphs generated with mixed-type or
766 // mixed-rank operations are likely to result in a disconnected network. Thus, we filter the
767 // operation signatures by primary data type and rank first, then generate random graph tests for
768 // each combination.
769 //
770 // Two parameterized tests are created for each filter:
771 // * 5-op graph with dimensions in range [1, 1000].
772 // * 40-op graph with dimensions in range [1, 10].
773 //
774 #define TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(dataType, rank) \
775 TEST_P(RandomGraphTest, SmallGraph_##dataType##_Rank##rank) { \
776 OperationFilter filter = {.dataTypes = {TestOperandType::dataType}, .ranks = {rank}}; \
777 OperationManager::get()->applyFilter(filter); \
778 mCriteria = kSmallGraphCriteria; \
779 testRandomGraph(GraphSize::SMALL, DimensionRange::WIDE); \
780 } \
781 TEST_P(RandomGraphTest, LargeGraph_##dataType##_Rank##rank) { \
782 OperationFilter filter = {.dataTypes = {TestOperandType::dataType}, .ranks = {rank}}; \
783 OperationManager::get()->applyFilter(filter); \
784 mCriteria = kLargeGraphCriteria; \
785 testRandomGraph(GraphSize::LARGE, DimensionRange::NARROW); \
786 }
787
788 // Random graph test with TENSOR_QUANT8_ASYMM as the primary data type is currently not defined.
789 // The generated graph with TENSOR_QUANT8_ASYMM as the primary data type will likely to result in
790 // disconnected graphs due to the mismatch between quantized parameters.
791
792 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 4);
793 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 3);
794 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 2);
795 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 1);
796
797 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 4);
798 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 3);
799 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 2);
800 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 1);
801
802 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 4);
803 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 3);
804 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 2);
805 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 1);
806
807 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 4);
808 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 3);
809 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 2);
810 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 1);
811
812 INSTANTIATE_TEST_CASE_P(TestRandomGraph, SingleOperationTest, ::testing::Range(0u, 50u));
813 INSTANTIATE_TEST_CASE_P(TestRandomGraph, RandomGraphTest, ::testing::Range(0u, 50u));
814
815 } // namespace fuzzing_test
816 } // namespace nn
817 } // namespace android
818