1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "GeneratedTestHarness.h"
18 
19 #include <android-base/logging.h>
20 #include <android/hardware/neuralnetworks/1.0/IDevice.h>
21 #include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
22 #include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
23 #include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
24 #include <android/hardware/neuralnetworks/1.0/types.h>
25 #include <android/hardware/neuralnetworks/1.1/IDevice.h>
26 #include <android/hardware/neuralnetworks/1.2/IDevice.h>
27 #include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
28 #include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
29 #include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
30 #include <android/hidl/allocator/1.0/IAllocator.h>
31 #include <android/hidl/memory/1.0/IMemory.h>
32 #include <gtest/gtest.h>
33 #include <hidlmemory/mapping.h>
34 
35 #include <algorithm>
36 #include <chrono>
37 #include <iostream>
38 #include <numeric>
39 #include <vector>
40 
41 #include "1.0/Utils.h"
42 #include "1.2/Callbacks.h"
43 #include "ExecutionBurstController.h"
44 #include "MemoryUtils.h"
45 #include "TestHarness.h"
46 #include "VtsHalNeuralnetworks.h"
47 
48 namespace android::hardware::neuralnetworks::V1_2::vts::functional {
49 
50 using namespace test_helper;
51 using hidl::memory::V1_0::IMemory;
52 using implementation::ExecutionCallback;
53 using implementation::PreparedModelCallback;
54 using V1_0::DataLocation;
55 using V1_0::ErrorStatus;
56 using V1_0::OperandLifeTime;
57 using V1_0::Request;
58 using V1_1::ExecutionPreference;
59 using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
60 
61 namespace {
62 
63 enum class Executor { ASYNC, SYNC, BURST };
64 
65 enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };
66 
67 struct TestConfig {
68     Executor executor;
69     MeasureTiming measureTiming;
70     OutputType outputType;
71     MemoryType memoryType;
72 };
73 
74 }  // namespace
75 
createModel(const TestModel & testModel)76 Model createModel(const TestModel& testModel) {
77     // Model operands.
78     CHECK_EQ(testModel.referenced.size(), 0u);  // Not supported in 1.1.
79     hidl_vec<Operand> operands(testModel.main.operands.size());
80     size_t constCopySize = 0, constRefSize = 0;
81     for (uint32_t i = 0; i < testModel.main.operands.size(); i++) {
82         const auto& op = testModel.main.operands[i];
83 
84         DataLocation loc = {};
85         if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
86             loc = {.poolIndex = 0,
87                    .offset = static_cast<uint32_t>(constCopySize),
88                    .length = static_cast<uint32_t>(op.data.size())};
89             constCopySize += op.data.alignedSize();
90         } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
91             loc = {.poolIndex = 0,
92                    .offset = static_cast<uint32_t>(constRefSize),
93                    .length = static_cast<uint32_t>(op.data.size())};
94             constRefSize += op.data.alignedSize();
95         }
96 
97         Operand::ExtraParams extraParams;
98         if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
99             extraParams.channelQuant(SymmPerChannelQuantParams{
100                     .scales = op.channelQuant.scales, .channelDim = op.channelQuant.channelDim});
101         }
102 
103         operands[i] = {.type = static_cast<OperandType>(op.type),
104                        .dimensions = op.dimensions,
105                        .numberOfConsumers = op.numberOfConsumers,
106                        .scale = op.scale,
107                        .zeroPoint = op.zeroPoint,
108                        .lifetime = static_cast<OperandLifeTime>(op.lifetime),
109                        .location = loc,
110                        .extraParams = std::move(extraParams)};
111     }
112 
113     // Model operations.
114     hidl_vec<Operation> operations(testModel.main.operations.size());
115     std::transform(testModel.main.operations.begin(), testModel.main.operations.end(),
116                    operations.begin(), [](const TestOperation& op) -> Operation {
117                        return {.type = static_cast<OperationType>(op.type),
118                                .inputs = op.inputs,
119                                .outputs = op.outputs};
120                    });
121 
122     // Constant copies.
123     hidl_vec<uint8_t> operandValues(constCopySize);
124     for (uint32_t i = 0; i < testModel.main.operands.size(); i++) {
125         const auto& op = testModel.main.operands[i];
126         if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
127             const uint8_t* begin = op.data.get<uint8_t>();
128             const uint8_t* end = begin + op.data.size();
129             std::copy(begin, end, operandValues.data() + operands[i].location.offset);
130         }
131     }
132 
133     // Shared memory.
134     hidl_vec<hidl_memory> pools = {};
135     if (constRefSize > 0) {
136         hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
137         CHECK_NE(pools[0].size(), 0u);
138 
139         // load data
140         sp<IMemory> mappedMemory = mapMemory(pools[0]);
141         CHECK(mappedMemory.get() != nullptr);
142         uint8_t* mappedPtr =
143                 reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
144         CHECK(mappedPtr != nullptr);
145 
146         for (uint32_t i = 0; i < testModel.main.operands.size(); i++) {
147             const auto& op = testModel.main.operands[i];
148             if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
149                 const uint8_t* begin = op.data.get<uint8_t>();
150                 const uint8_t* end = begin + op.data.size();
151                 std::copy(begin, end, mappedPtr + operands[i].location.offset);
152             }
153         }
154     }
155 
156     return {.operands = std::move(operands),
157             .operations = std::move(operations),
158             .inputIndexes = testModel.main.inputIndexes,
159             .outputIndexes = testModel.main.outputIndexes,
160             .operandValues = std::move(operandValues),
161             .pools = std::move(pools),
162             .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
163 }
164 
isOutputSizeGreaterThanOne(const TestModel & testModel,uint32_t index)165 static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) {
166     const auto byteSize = testModel.main.operands[testModel.main.outputIndexes[index]].data.size();
167     return byteSize > 1u;
168 }
169 
makeOutputInsufficientSize(uint32_t outputIndex,Request * request)170 static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) {
171     auto& length = request->outputs[outputIndex].location.length;
172     ASSERT_GT(length, 1u);
173     length -= 1u;
174 }
175 
makeOutputDimensionsUnspecified(Model * model)176 static void makeOutputDimensionsUnspecified(Model* model) {
177     for (auto i : model->outputIndexes) {
178         auto& dims = model->operands[i].dimensions;
179         std::fill(dims.begin(), dims.end(), 0);
180     }
181 }
182 
ExecutePreparedModel(const sp<IPreparedModel> & preparedModel,const Request & request,MeasureTiming measure,sp<ExecutionCallback> & callback)183 static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
184                                                 const Request& request, MeasureTiming measure,
185                                                 sp<ExecutionCallback>& callback) {
186     return preparedModel->execute_1_2(request, measure, callback);
187 }
ExecutePreparedModel(const sp<IPreparedModel> & preparedModel,const Request & request,MeasureTiming measure,hidl_vec<OutputShape> * outputShapes,Timing * timing)188 static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
189                                                 const Request& request, MeasureTiming measure,
190                                                 hidl_vec<OutputShape>* outputShapes,
191                                                 Timing* timing) {
192     ErrorStatus result;
193     Return<void> ret = preparedModel->executeSynchronously(
194             request, measure,
195             [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
196                                             const Timing& time) {
197                 result = error;
198                 *outputShapes = shapes;
199                 *timing = time;
200             });
201     if (!ret.isOk()) {
202         return ErrorStatus::GENERAL_FAILURE;
203     }
204     return result;
205 }
CreateBurst(const sp<IPreparedModel> & preparedModel)206 static std::shared_ptr<::android::nn::ExecutionBurstController> CreateBurst(
207         const sp<IPreparedModel>& preparedModel) {
208     return android::nn::ExecutionBurstController::create(preparedModel,
209                                                          std::chrono::microseconds{0});
210 }
211 
EvaluatePreparedModel(const sp<IPreparedModel> & preparedModel,const TestModel & testModel,const TestConfig & testConfig)212 void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
213                            const TestConfig& testConfig) {
214     // If output0 does not have size larger than one byte, we can not test with insufficient buffer.
215     if (testConfig.outputType == OutputType::INSUFFICIENT &&
216         !isOutputSizeGreaterThanOne(testModel, 0)) {
217         return;
218     }
219 
220     ExecutionContext context;
221     Request request = context.createRequest(testModel, testConfig.memoryType);
222     if (testConfig.outputType == OutputType::INSUFFICIENT) {
223         makeOutputInsufficientSize(/*outputIndex=*/0, &request);
224     }
225 
226     ErrorStatus executionStatus;
227     hidl_vec<OutputShape> outputShapes;
228     Timing timing;
229     switch (testConfig.executor) {
230         case Executor::ASYNC: {
231             SCOPED_TRACE("asynchronous");
232 
233             // launch execution
234             sp<ExecutionCallback> executionCallback = new ExecutionCallback();
235             Return<ErrorStatus> executionLaunchStatus = ExecutePreparedModel(
236                     preparedModel, request, testConfig.measureTiming, executionCallback);
237             ASSERT_TRUE(executionLaunchStatus.isOk());
238             EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
239 
240             // retrieve execution status
241             executionCallback->wait();
242             executionStatus = executionCallback->getStatus();
243             outputShapes = executionCallback->getOutputShapes();
244             timing = executionCallback->getTiming();
245 
246             break;
247         }
248         case Executor::SYNC: {
249             SCOPED_TRACE("synchronous");
250 
251             // execute
252             Return<ErrorStatus> executionReturnStatus = ExecutePreparedModel(
253                     preparedModel, request, testConfig.measureTiming, &outputShapes, &timing);
254             ASSERT_TRUE(executionReturnStatus.isOk());
255             executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
256 
257             break;
258         }
259         case Executor::BURST: {
260             SCOPED_TRACE("burst");
261 
262             // create burst
263             const std::shared_ptr<::android::nn::ExecutionBurstController> controller =
264                     CreateBurst(preparedModel);
265             ASSERT_NE(nullptr, controller.get());
266 
267             // create memory keys
268             std::vector<intptr_t> keys(request.pools.size());
269             for (size_t i = 0; i < keys.size(); ++i) {
270                 keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
271             }
272 
273             // execute burst
274             int n;
275             std::tie(n, outputShapes, timing, std::ignore) =
276                     controller->compute(request, testConfig.measureTiming, keys);
277             executionStatus = nn::legacyConvertResultCodeToErrorStatus(n);
278 
279             break;
280         }
281     }
282 
283     if (testConfig.outputType != OutputType::FULLY_SPECIFIED &&
284         executionStatus == ErrorStatus::GENERAL_FAILURE) {
285         LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
286                      "execute model that it does not support.";
287         std::cout << "[          ]   Early termination of test because vendor service cannot "
288                      "execute model that it does not support."
289                   << std::endl;
290         GTEST_SKIP();
291     }
292     if (testConfig.measureTiming == MeasureTiming::NO) {
293         EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
294         EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
295     } else {
296         if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
297             EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
298         }
299     }
300 
301     switch (testConfig.outputType) {
302         case OutputType::FULLY_SPECIFIED:
303             // If the model output operands are fully specified, outputShapes must be either
304             // either empty, or have the same number of elements as the number of outputs.
305             ASSERT_EQ(ErrorStatus::NONE, executionStatus);
306             ASSERT_TRUE(outputShapes.size() == 0 ||
307                         outputShapes.size() == testModel.main.outputIndexes.size());
308             break;
309         case OutputType::UNSPECIFIED:
310             // If the model output operands are not fully specified, outputShapes must have
311             // the same number of elements as the number of outputs.
312             ASSERT_EQ(ErrorStatus::NONE, executionStatus);
313             ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size());
314             break;
315         case OutputType::INSUFFICIENT:
316             ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus);
317             ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size());
318             ASSERT_FALSE(outputShapes[0].isSufficient);
319             return;
320     }
321 
322     // Go through all outputs, check returned output shapes.
323     for (uint32_t i = 0; i < outputShapes.size(); i++) {
324         EXPECT_TRUE(outputShapes[i].isSufficient);
325         const auto& expect = testModel.main.operands[testModel.main.outputIndexes[i]].dimensions;
326         const std::vector<uint32_t> actual = outputShapes[i].dimensions;
327         EXPECT_EQ(expect, actual);
328     }
329 
330     // Retrieve execution results.
331     const std::vector<TestBuffer> outputs = context.getOutputBuffers(request);
332 
333     // We want "close-enough" results.
334     checkResults(testModel, outputs);
335 }
336 
EvaluatePreparedModel(const sp<IPreparedModel> & preparedModel,const TestModel & testModel,bool testDynamicOutputShape)337 void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
338                            bool testDynamicOutputShape) {
339     std::vector<OutputType> outputTypesList;
340     std::vector<MeasureTiming> measureTimingList;
341     std::vector<Executor> executorList;
342     std::vector<MemoryType> memoryTypeList;
343 
344     if (testDynamicOutputShape) {
345         outputTypesList = {OutputType::UNSPECIFIED, OutputType::INSUFFICIENT};
346         measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
347         executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST};
348         memoryTypeList = {MemoryType::ASHMEM};
349     } else {
350         outputTypesList = {OutputType::FULLY_SPECIFIED};
351         measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
352         executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST};
353         memoryTypeList = {MemoryType::ASHMEM};
354     }
355 
356     for (const OutputType outputType : outputTypesList) {
357         for (const MeasureTiming measureTiming : measureTimingList) {
358             for (const Executor executor : executorList) {
359                 for (const MemoryType memoryType : memoryTypeList) {
360                     const TestConfig testConfig = {.executor = executor,
361                                                    .measureTiming = measureTiming,
362                                                    .outputType = outputType,
363                                                    .memoryType = memoryType};
364                     EvaluatePreparedModel(preparedModel, testModel, testConfig);
365                 }
366             }
367         }
368     }
369 }
370 
Execute(const sp<IDevice> & device,const TestModel & testModel,bool testDynamicOutputShape)371 void Execute(const sp<IDevice>& device, const TestModel& testModel, bool testDynamicOutputShape) {
372     Model model = createModel(testModel);
373     if (testDynamicOutputShape) {
374         makeOutputDimensionsUnspecified(&model);
375     }
376 
377     sp<IPreparedModel> preparedModel;
378     createPreparedModel(device, model, &preparedModel);
379     if (preparedModel == nullptr) return;
380 
381     EvaluatePreparedModel(preparedModel, testModel, testDynamicOutputShape);
382 }
383 
SetUp()384 void GeneratedTestBase::SetUp() {
385     testing::TestWithParam<GeneratedTestParam>::SetUp();
386     ASSERT_NE(kDevice, nullptr);
387     const bool deviceIsResponsive = kDevice->ping().isOk();
388     ASSERT_TRUE(deviceIsResponsive);
389 }
390 
getNamedModels(const FilterFn & filter)391 std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
392     return TestModelManager::get().getTestModels(filter);
393 }
394 
getNamedModels(const FilterNameFn & filter)395 std::vector<NamedModel> getNamedModels(const FilterNameFn& filter) {
396     return TestModelManager::get().getTestModels(filter);
397 }
398 
printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam> & info)399 std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
400     const auto& [namedDevice, namedModel] = info.param;
401     return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
402 }
403 
404 // Tag for the generated tests
405 class GeneratedTest : public GeneratedTestBase {};
406 
407 // Tag for the dynamic output shape tests
408 class DynamicOutputShapeTest : public GeneratedTest {};
409 
TEST_P(GeneratedTest,Test)410 TEST_P(GeneratedTest, Test) {
411     Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/false);
412 }
413 
TEST_P(DynamicOutputShapeTest,Test)414 TEST_P(DynamicOutputShapeTest, Test) {
415     Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/true);
416 }
417 
418 INSTANTIATE_GENERATED_TEST(GeneratedTest,
__anon2d17c2040402(const TestModel& testModel) 419                            [](const TestModel& testModel) { return !testModel.expectFailure; });
420 
421 INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest,
__anon2d17c2040502(const TestModel& testModel) 422                            [](const TestModel& testModel) { return !testModel.expectFailure; });
423 
424 }  // namespace android::hardware::neuralnetworks::V1_2::vts::functional
425