1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "neuralnetworks_hidl_hal_test"
18
19 #include "VtsHalNeuralnetworks.h"
20
21 #include "1.2/Callbacks.h"
22 #include "ExecutionBurstController.h"
23 #include "ExecutionBurstServer.h"
24 #include "GeneratedTestHarness.h"
25 #include "TestHarness.h"
26
27 #include <android-base/logging.h>
28 #include <chrono>
29 #include <cstring>
30
31 namespace android::hardware::neuralnetworks::V1_2::vts::functional {
32
33 using nn::ExecutionBurstController;
34 using nn::RequestChannelSender;
35 using nn::ResultChannelReceiver;
36 using V1_0::ErrorStatus;
37 using V1_0::Request;
38 using ExecutionBurstCallback = ExecutionBurstController::ExecutionBurstCallback;
39
40 using BurstExecutionMutation = std::function<void(std::vector<FmqRequestDatum>*)>;
41
42 // This constant value represents the length of an FMQ that is large enough to
43 // return a result from a burst execution for all of the generated test cases.
44 constexpr size_t kExecutionBurstChannelLength = 1024;
45
46 // This constant value represents a length of an FMQ that is not large enough
47 // to return a result from a burst execution for some of the generated test
48 // cases.
49 constexpr size_t kExecutionBurstChannelSmallLength = 8;
50
51 ///////////////////////// UTILITY FUNCTIONS /////////////////////////
52
badTiming(Timing timing)53 static bool badTiming(Timing timing) {
54 return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX;
55 }
56
createBurst(const sp<IPreparedModel> & preparedModel,const sp<IBurstCallback> & callback,std::unique_ptr<RequestChannelSender> * sender,std::unique_ptr<ResultChannelReceiver> * receiver,sp<IBurstContext> * context,size_t resultChannelLength=kExecutionBurstChannelLength)57 static void createBurst(const sp<IPreparedModel>& preparedModel, const sp<IBurstCallback>& callback,
58 std::unique_ptr<RequestChannelSender>* sender,
59 std::unique_ptr<ResultChannelReceiver>* receiver,
60 sp<IBurstContext>* context,
61 size_t resultChannelLength = kExecutionBurstChannelLength) {
62 ASSERT_NE(nullptr, preparedModel.get());
63 ASSERT_NE(nullptr, sender);
64 ASSERT_NE(nullptr, receiver);
65 ASSERT_NE(nullptr, context);
66
67 // create FMQ objects
68 auto [fmqRequestChannel, fmqRequestDescriptor] =
69 RequestChannelSender::create(kExecutionBurstChannelLength);
70 auto [fmqResultChannel, fmqResultDescriptor] =
71 ResultChannelReceiver::create(resultChannelLength, std::chrono::microseconds{0});
72 ASSERT_NE(nullptr, fmqRequestChannel.get());
73 ASSERT_NE(nullptr, fmqResultChannel.get());
74 ASSERT_NE(nullptr, fmqRequestDescriptor);
75 ASSERT_NE(nullptr, fmqResultDescriptor);
76
77 // configure burst
78 ErrorStatus errorStatus;
79 sp<IBurstContext> burstContext;
80 const Return<void> ret = preparedModel->configureExecutionBurst(
81 callback, *fmqRequestDescriptor, *fmqResultDescriptor,
82 [&errorStatus, &burstContext](ErrorStatus status, const sp<IBurstContext>& context) {
83 errorStatus = status;
84 burstContext = context;
85 });
86 ASSERT_TRUE(ret.isOk());
87 ASSERT_EQ(ErrorStatus::NONE, errorStatus);
88 ASSERT_NE(nullptr, burstContext.get());
89
90 // return values
91 *sender = std::move(fmqRequestChannel);
92 *receiver = std::move(fmqResultChannel);
93 *context = burstContext;
94 }
95
createBurstWithResultChannelLength(const sp<IPreparedModel> & preparedModel,size_t resultChannelLength,std::shared_ptr<ExecutionBurstController> * controller)96 static void createBurstWithResultChannelLength(
97 const sp<IPreparedModel>& preparedModel, size_t resultChannelLength,
98 std::shared_ptr<ExecutionBurstController>* controller) {
99 ASSERT_NE(nullptr, preparedModel.get());
100 ASSERT_NE(nullptr, controller);
101
102 // create FMQ objects
103 std::unique_ptr<RequestChannelSender> sender;
104 std::unique_ptr<ResultChannelReceiver> receiver;
105 sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
106 sp<IBurstContext> context;
107 ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context,
108 resultChannelLength));
109 ASSERT_NE(nullptr, sender.get());
110 ASSERT_NE(nullptr, receiver.get());
111 ASSERT_NE(nullptr, context.get());
112
113 // return values
114 *controller = std::make_shared<ExecutionBurstController>(std::move(sender), std::move(receiver),
115 context, callback);
116 }
117
118 // Primary validation function. This function will take a valid serialized
119 // request, apply a mutation to it to invalidate the serialized request, then
120 // pass it to interface calls that use the serialized request.
validate(RequestChannelSender * sender,ResultChannelReceiver * receiver,const std::string & message,const std::vector<FmqRequestDatum> & originalSerialized,const BurstExecutionMutation & mutate)121 static void validate(RequestChannelSender* sender, ResultChannelReceiver* receiver,
122 const std::string& message,
123 const std::vector<FmqRequestDatum>& originalSerialized,
124 const BurstExecutionMutation& mutate) {
125 std::vector<FmqRequestDatum> serialized = originalSerialized;
126 mutate(&serialized);
127
128 // skip if packet is too large to send
129 if (serialized.size() > kExecutionBurstChannelLength) {
130 return;
131 }
132
133 SCOPED_TRACE(message);
134
135 // send invalid packet
136 ASSERT_TRUE(sender->sendPacket(serialized));
137
138 // receive error
139 auto results = receiver->getBlocking();
140 ASSERT_TRUE(results.has_value());
141 const auto [status, outputShapes, timing] = std::move(*results);
142 EXPECT_NE(ErrorStatus::NONE, status);
143 EXPECT_EQ(0u, outputShapes.size());
144 EXPECT_TRUE(badTiming(timing));
145 }
146
147 // For validation, valid packet entries are mutated to invalid packet entries,
148 // or invalid packet entries are inserted into valid packets. This function
149 // creates pre-set invalid packet entries for convenience.
createBadRequestPacketEntries()150 static std::vector<FmqRequestDatum> createBadRequestPacketEntries() {
151 const FmqRequestDatum::PacketInformation packetInformation = {
152 /*.packetSize=*/10, /*.numberOfInputOperands=*/10, /*.numberOfOutputOperands=*/10,
153 /*.numberOfPools=*/10};
154 const FmqRequestDatum::OperandInformation operandInformation = {
155 /*.hasNoValue=*/false, /*.location=*/{}, /*.numberOfDimensions=*/10};
156 const int32_t invalidPoolIdentifier = std::numeric_limits<int32_t>::max();
157 std::vector<FmqRequestDatum> bad(7);
158 bad[0].packetInformation(packetInformation);
159 bad[1].inputOperandInformation(operandInformation);
160 bad[2].inputOperandDimensionValue(0);
161 bad[3].outputOperandInformation(operandInformation);
162 bad[4].outputOperandDimensionValue(0);
163 bad[5].poolIdentifier(invalidPoolIdentifier);
164 bad[6].measureTiming(MeasureTiming::YES);
165 return bad;
166 }
167
168 // For validation, valid packet entries are mutated to invalid packet entries,
169 // or invalid packet entries are inserted into valid packets. This function
170 // retrieves pre-set invalid packet entries for convenience. This function
171 // caches these data so they can be reused on subsequent validation checks.
getBadRequestPacketEntries()172 static const std::vector<FmqRequestDatum>& getBadRequestPacketEntries() {
173 static const std::vector<FmqRequestDatum> bad = createBadRequestPacketEntries();
174 return bad;
175 }
176
177 ///////////////////////// REMOVE DATUM ////////////////////////////////////
178
removeDatumTest(RequestChannelSender * sender,ResultChannelReceiver * receiver,const std::vector<FmqRequestDatum> & serialized)179 static void removeDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
180 const std::vector<FmqRequestDatum>& serialized) {
181 for (size_t index = 0; index < serialized.size(); ++index) {
182 const std::string message = "removeDatum: removed datum at index " + std::to_string(index);
183 validate(sender, receiver, message, serialized,
184 [index](std::vector<FmqRequestDatum>* serialized) {
185 serialized->erase(serialized->begin() + index);
186 });
187 }
188 }
189
190 ///////////////////////// ADD DATUM ////////////////////////////////////
191
addDatumTest(RequestChannelSender * sender,ResultChannelReceiver * receiver,const std::vector<FmqRequestDatum> & serialized)192 static void addDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
193 const std::vector<FmqRequestDatum>& serialized) {
194 const std::vector<FmqRequestDatum>& extra = getBadRequestPacketEntries();
195 for (size_t index = 0; index <= serialized.size(); ++index) {
196 for (size_t type = 0; type < extra.size(); ++type) {
197 const std::string message = "addDatum: added datum type " + std::to_string(type) +
198 " at index " + std::to_string(index);
199 validate(sender, receiver, message, serialized,
200 [index, type, &extra](std::vector<FmqRequestDatum>* serialized) {
201 serialized->insert(serialized->begin() + index, extra[type]);
202 });
203 }
204 }
205 }
206
207 ///////////////////////// MUTATE DATUM ////////////////////////////////////
208
interestingCase(const FmqRequestDatum & lhs,const FmqRequestDatum & rhs)209 static bool interestingCase(const FmqRequestDatum& lhs, const FmqRequestDatum& rhs) {
210 using Discriminator = FmqRequestDatum::hidl_discriminator;
211
212 const bool differentValues = (lhs != rhs);
213 const bool sameDiscriminator = (lhs.getDiscriminator() == rhs.getDiscriminator());
214 const auto discriminator = rhs.getDiscriminator();
215 const bool isDimensionValue = (discriminator == Discriminator::inputOperandDimensionValue ||
216 discriminator == Discriminator::outputOperandDimensionValue);
217
218 return differentValues && !(sameDiscriminator && isDimensionValue);
219 }
220
mutateDatumTest(RequestChannelSender * sender,ResultChannelReceiver * receiver,const std::vector<FmqRequestDatum> & serialized)221 static void mutateDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
222 const std::vector<FmqRequestDatum>& serialized) {
223 const std::vector<FmqRequestDatum>& change = getBadRequestPacketEntries();
224 for (size_t index = 0; index < serialized.size(); ++index) {
225 for (size_t type = 0; type < change.size(); ++type) {
226 if (interestingCase(serialized[index], change[type])) {
227 const std::string message = "mutateDatum: changed datum at index " +
228 std::to_string(index) + " to datum type " +
229 std::to_string(type);
230 validate(sender, receiver, message, serialized,
231 [index, type, &change](std::vector<FmqRequestDatum>* serialized) {
232 (*serialized)[index] = change[type];
233 });
234 }
235 }
236 }
237 }
238
239 ///////////////////////// BURST VALIATION TESTS ////////////////////////////////////
240
validateBurstSerialization(const sp<IPreparedModel> & preparedModel,const Request & request)241 static void validateBurstSerialization(const sp<IPreparedModel>& preparedModel,
242 const Request& request) {
243 // create burst
244 std::unique_ptr<RequestChannelSender> sender;
245 std::unique_ptr<ResultChannelReceiver> receiver;
246 sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
247 sp<IBurstContext> context;
248 ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
249 ASSERT_NE(nullptr, sender.get());
250 ASSERT_NE(nullptr, receiver.get());
251 ASSERT_NE(nullptr, context.get());
252
253 // load memory into callback slots
254 std::vector<intptr_t> keys;
255 keys.reserve(request.pools.size());
256 std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
257 [](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
258 const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
259
260 // ensure slot std::numeric_limits<int32_t>::max() doesn't exist (for
261 // subsequent slot validation testing)
262 ASSERT_TRUE(std::all_of(slots.begin(), slots.end(), [](int32_t slot) {
263 return slot != std::numeric_limits<int32_t>::max();
264 }));
265
266 // serialize the request
267 const auto serialized = android::nn::serialize(request, MeasureTiming::YES, slots);
268
269 // validations
270 removeDatumTest(sender.get(), receiver.get(), serialized);
271 addDatumTest(sender.get(), receiver.get(), serialized);
272 mutateDatumTest(sender.get(), receiver.get(), serialized);
273 }
274
275 // This test validates that when the Result message size exceeds length of the
276 // result FMQ, the service instance gracefully fails and returns an error.
validateBurstFmqLength(const sp<IPreparedModel> & preparedModel,const Request & request)277 static void validateBurstFmqLength(const sp<IPreparedModel>& preparedModel,
278 const Request& request) {
279 // create regular burst
280 std::shared_ptr<ExecutionBurstController> controllerRegular;
281 ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
282 preparedModel, kExecutionBurstChannelLength, &controllerRegular));
283 ASSERT_NE(nullptr, controllerRegular.get());
284
285 // create burst with small output channel
286 std::shared_ptr<ExecutionBurstController> controllerSmall;
287 ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
288 preparedModel, kExecutionBurstChannelSmallLength, &controllerSmall));
289 ASSERT_NE(nullptr, controllerSmall.get());
290
291 // load memory into callback slots
292 std::vector<intptr_t> keys(request.pools.size());
293 for (size_t i = 0; i < keys.size(); ++i) {
294 keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
295 }
296
297 // collect serialized result by running regular burst
298 const auto [nRegular, outputShapesRegular, timingRegular, fallbackRegular] =
299 controllerRegular->compute(request, MeasureTiming::NO, keys);
300 const ErrorStatus statusRegular = nn::legacyConvertResultCodeToErrorStatus(nRegular);
301 EXPECT_FALSE(fallbackRegular);
302
303 // skip test if regular burst output isn't useful for testing a failure
304 // caused by having too small of a length for the result FMQ
305 const std::vector<FmqResultDatum> serialized =
306 android::nn::serialize(statusRegular, outputShapesRegular, timingRegular);
307 if (statusRegular != ErrorStatus::NONE ||
308 serialized.size() <= kExecutionBurstChannelSmallLength) {
309 return;
310 }
311
312 // by this point, execution should fail because the result channel isn't
313 // large enough to return the serialized result
314 const auto [nSmall, outputShapesSmall, timingSmall, fallbackSmall] =
315 controllerSmall->compute(request, MeasureTiming::NO, keys);
316 const ErrorStatus statusSmall = nn::legacyConvertResultCodeToErrorStatus(nSmall);
317 EXPECT_NE(ErrorStatus::NONE, statusSmall);
318 EXPECT_EQ(0u, outputShapesSmall.size());
319 EXPECT_TRUE(badTiming(timingSmall));
320 EXPECT_FALSE(fallbackSmall);
321 }
322
isSanitized(const FmqResultDatum & datum)323 static bool isSanitized(const FmqResultDatum& datum) {
324 using Discriminator = FmqResultDatum::hidl_discriminator;
325
326 // check to ensure the padding values in the returned
327 // FmqResultDatum::OperandInformation are initialized to 0
328 if (datum.getDiscriminator() == Discriminator::operandInformation) {
329 static_assert(
330 offsetof(FmqResultDatum::OperandInformation, isSufficient) == 0,
331 "unexpected value for offset of FmqResultDatum::OperandInformation::isSufficient");
332 static_assert(
333 sizeof(FmqResultDatum::OperandInformation::isSufficient) == 1,
334 "unexpected value for size of FmqResultDatum::OperandInformation::isSufficient");
335 static_assert(offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) == 4,
336 "unexpected value for offset of "
337 "FmqResultDatum::OperandInformation::numberOfDimensions");
338 static_assert(sizeof(FmqResultDatum::OperandInformation::numberOfDimensions) == 4,
339 "unexpected value for size of "
340 "FmqResultDatum::OperandInformation::numberOfDimensions");
341 static_assert(sizeof(FmqResultDatum::OperandInformation) == 8,
342 "unexpected value for size of "
343 "FmqResultDatum::OperandInformation");
344
345 constexpr size_t paddingOffset =
346 offsetof(FmqResultDatum::OperandInformation, isSufficient) +
347 sizeof(FmqResultDatum::OperandInformation::isSufficient);
348 constexpr size_t paddingSize =
349 offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) - paddingOffset;
350
351 FmqResultDatum::OperandInformation initialized{};
352 std::memset(&initialized, 0, sizeof(initialized));
353
354 const char* initializedPaddingStart =
355 reinterpret_cast<const char*>(&initialized) + paddingOffset;
356 const char* datumPaddingStart =
357 reinterpret_cast<const char*>(&datum.operandInformation()) + paddingOffset;
358
359 return std::memcmp(datumPaddingStart, initializedPaddingStart, paddingSize) == 0;
360 }
361
362 // there are no other padding initialization checks required, so return true
363 // for any sum-type that isn't FmqResultDatum::OperandInformation
364 return true;
365 }
366
validateBurstSanitized(const sp<IPreparedModel> & preparedModel,const Request & request)367 static void validateBurstSanitized(const sp<IPreparedModel>& preparedModel,
368 const Request& request) {
369 // create burst
370 std::unique_ptr<RequestChannelSender> sender;
371 std::unique_ptr<ResultChannelReceiver> receiver;
372 sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
373 sp<IBurstContext> context;
374 ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
375 ASSERT_NE(nullptr, sender.get());
376 ASSERT_NE(nullptr, receiver.get());
377 ASSERT_NE(nullptr, context.get());
378
379 // load memory into callback slots
380 std::vector<intptr_t> keys;
381 keys.reserve(request.pools.size());
382 std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
383 [](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
384 const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
385
386 // send valid request
387 ASSERT_TRUE(sender->send(request, MeasureTiming::YES, slots));
388
389 // receive valid result
390 auto serialized = receiver->getPacketBlocking();
391 ASSERT_TRUE(serialized.has_value());
392
393 // sanitize result
394 ASSERT_TRUE(std::all_of(serialized->begin(), serialized->end(), isSanitized))
395 << "The result serialized data is not properly sanitized";
396 }
397
398 ///////////////////////////// ENTRY POINT //////////////////////////////////
399
validateBurst(const sp<IPreparedModel> & preparedModel,const Request & request)400 void validateBurst(const sp<IPreparedModel>& preparedModel, const Request& request) {
401 ASSERT_NO_FATAL_FAILURE(validateBurstSerialization(preparedModel, request));
402 ASSERT_NO_FATAL_FAILURE(validateBurstFmqLength(preparedModel, request));
403 ASSERT_NO_FATAL_FAILURE(validateBurstSanitized(preparedModel, request));
404 }
405
406 } // namespace android::hardware::neuralnetworks::V1_2::vts::functional
407