1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "neuralnetworks_aidl_hal_test"
18 #include "VtsHalNeuralnetworks.h"
19
20 #include <android-base/logging.h>
21 #include <android/binder_auto_utils.h>
22 #include <android/binder_interface_utils.h>
23 #include <android/binder_manager.h>
24 #include <android/binder_status.h>
25 #include <gtest/gtest.h>
26 #include <memory>
27 #include <string>
28 #include <utility>
29
30 #include <TestHarness.h>
31 #include <aidl/Vintf.h>
32 #include <nnapi/hal/aidl/Conversions.h>
33
34 #include "Callbacks.h"
35 #include "GeneratedTestHarness.h"
36 #include "Utils.h"
37
38 namespace aidl::android::hardware::neuralnetworks::vts::functional {
39
40 using implementation::PreparedModelCallback;
41
42 // internal helper function
createPreparedModel(const std::shared_ptr<IDevice> & device,const Model & model,std::shared_ptr<IPreparedModel> * preparedModel,bool reportSkipping)43 void createPreparedModel(const std::shared_ptr<IDevice>& device, const Model& model,
44 std::shared_ptr<IPreparedModel>* preparedModel, bool reportSkipping) {
45 ASSERT_NE(nullptr, preparedModel);
46 *preparedModel = nullptr;
47
48 // see if service can handle model
49 std::vector<bool> supportedOperations;
50 const auto supportedCallStatus = device->getSupportedOperations(model, &supportedOperations);
51 ASSERT_TRUE(supportedCallStatus.isOk());
52 ASSERT_NE(0ul, supportedOperations.size());
53 const bool fullySupportsModel = std::all_of(
54 supportedOperations.begin(), supportedOperations.end(), [](bool v) { return v; });
55
56 // launch prepare model
57 const std::shared_ptr<PreparedModelCallback> preparedModelCallback =
58 ndk::SharedRefBase::make<PreparedModelCallback>();
59 const auto prepareLaunchStatus =
60 device->prepareModel(model, ExecutionPreference::FAST_SINGLE_ANSWER, kDefaultPriority,
61 kNoDeadline, {}, {}, kEmptyCacheToken, preparedModelCallback);
62 ASSERT_TRUE(prepareLaunchStatus.isOk()) << prepareLaunchStatus.getDescription();
63
64 // retrieve prepared model
65 preparedModelCallback->wait();
66 const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
67 *preparedModel = preparedModelCallback->getPreparedModel();
68
69 // The getSupportedOperations call returns a list of operations that are guaranteed not to fail
70 // if prepareModel is called, and 'fullySupportsModel' is true i.f.f. the entire model is
71 // guaranteed. If a driver has any doubt that it can prepare an operation, it must return false.
72 // So here, if a driver isn't sure if it can support an operation, but reports that it
73 // successfully prepared the model, the test can continue.
74 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
75 ASSERT_EQ(nullptr, preparedModel->get());
76 if (!reportSkipping) {
77 return;
78 }
79 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot prepare "
80 "model that it does not support.";
81 std::cout << "[ ] Early termination of test because vendor service cannot "
82 "prepare model that it does not support."
83 << std::endl;
84 GTEST_SKIP();
85 }
86
87 ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);
88 ASSERT_NE(nullptr, preparedModel->get());
89 }
90
SetUp()91 void NeuralNetworksAidlTest::SetUp() {
92 testing::TestWithParam<NeuralNetworksAidlTestParam>::SetUp();
93 ASSERT_NE(kDevice, nullptr);
94 const bool deviceIsResponsive =
95 ndk::ScopedAStatus::fromStatus(AIBinder_ping(kDevice->asBinder().get())).isOk();
96 ASSERT_TRUE(deviceIsResponsive);
97 }
98
makeNamedDevice(const std::string & name)99 static NamedDevice makeNamedDevice(const std::string& name) {
100 ndk::SpAIBinder binder(AServiceManager_waitForService(name.c_str()));
101 return {name, IDevice::fromBinder(binder)};
102 }
103
getNamedDevicesImpl()104 static std::vector<NamedDevice> getNamedDevicesImpl() {
105 // Retrieves the name of all service instances that implement IDevice,
106 // including any Lazy HAL instances.
107 const std::vector<std::string> names = ::android::getAidlHalInstanceNames(IDevice::descriptor);
108
109 // Get a handle to each device and pair it with its name.
110 std::vector<NamedDevice> namedDevices;
111 namedDevices.reserve(names.size());
112 std::transform(names.begin(), names.end(), std::back_inserter(namedDevices), makeNamedDevice);
113 return namedDevices;
114 }
115
getNamedDevices()116 const std::vector<NamedDevice>& getNamedDevices() {
117 const static std::vector<NamedDevice> devices = getNamedDevicesImpl();
118 return devices;
119 }
120
printNeuralNetworksAidlTest(const testing::TestParamInfo<NeuralNetworksAidlTestParam> & info)121 std::string printNeuralNetworksAidlTest(
122 const testing::TestParamInfo<NeuralNetworksAidlTestParam>& info) {
123 return gtestCompliantName(getName(info.param));
124 }
125
126 INSTANTIATE_DEVICE_TEST(NeuralNetworksAidlTest);
127
128 // Forward declaration from ValidateModel.cpp
129 void validateModel(const std::shared_ptr<IDevice>& device, const Model& model);
130 // Forward declaration from ValidateRequest.cpp
131 void validateRequest(const std::shared_ptr<IPreparedModel>& preparedModel, const Request& request);
132 // Forward declaration from ValidateRequest.cpp
133 void validateBurst(const std::shared_ptr<IPreparedModel>& preparedModel, const Request& request);
134 // Forward declaration from ValidateRequest.cpp
135 void validateRequestFailure(const std::shared_ptr<IPreparedModel>& preparedModel,
136 const Request& request);
137
validateEverything(const std::shared_ptr<IDevice> & device,const Model & model,const Request & request)138 void validateEverything(const std::shared_ptr<IDevice>& device, const Model& model,
139 const Request& request) {
140 validateModel(device, model);
141
142 // Create IPreparedModel.
143 std::shared_ptr<IPreparedModel> preparedModel;
144 createPreparedModel(device, model, &preparedModel);
145 if (preparedModel == nullptr) return;
146
147 validateRequest(preparedModel, request);
148 validateBurst(preparedModel, request);
149 // HIDL also had test that expected executeFenced to fail on received null fd (-1). This is not
150 // allowed in AIDL and will result in EX_TRANSACTION_FAILED.
151 }
152
validateFailure(const std::shared_ptr<IDevice> & device,const Model & model,const Request & request)153 void validateFailure(const std::shared_ptr<IDevice>& device, const Model& model,
154 const Request& request) {
155 // TODO: Should this always succeed?
156 // What if the invalid input is part of the model (i.e., a parameter).
157 validateModel(device, model);
158
159 // Create IPreparedModel.
160 std::shared_ptr<IPreparedModel> preparedModel;
161 createPreparedModel(device, model, &preparedModel);
162 if (preparedModel == nullptr) return;
163
164 validateRequestFailure(preparedModel, request);
165 }
166
TEST_P(ValidationTest,Test)167 TEST_P(ValidationTest, Test) {
168 const Model model = createModel(kTestModel);
169 ExecutionContext context;
170 const Request request = context.createRequest(kTestModel);
171 if (kTestModel.expectFailure) {
172 validateFailure(kDevice, model, request);
173 } else {
174 validateEverything(kDevice, model, request);
175 }
176 }
177
__anon0ec80da00202(const std::string& testName) 178 INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) {
179 // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs"
180 // generated tests.
181 return testName.find("inputs_as_internal") == std::string::npos &&
182 testName.find("all_tensors_as_inputs") == std::string::npos;
183 });
184
toString(Executor executor)185 std::string toString(Executor executor) {
186 switch (executor) {
187 case Executor::SYNC:
188 return "SYNC";
189 case Executor::BURST:
190 return "BURST";
191 case Executor::FENCED:
192 return "FENCED";
193 default:
194 CHECK(false);
195 }
196 }
197
198 } // namespace aidl::android::hardware::neuralnetworks::vts::functional
199