1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "PreparedModel.h"
18
19 #include "Callbacks.h"
20 #include "Conversions.h"
21 #include "Execution.h"
22 #include "Utils.h"
23
24 #include <android/hardware/neuralnetworks/1.0/types.h>
25 #include <android/hardware/neuralnetworks/1.1/types.h>
26 #include <android/hardware/neuralnetworks/1.2/types.h>
27 #include <android/hardware/neuralnetworks/1.3/IPreparedModel.h>
28 #include <android/hardware/neuralnetworks/1.3/types.h>
29 #include <nnapi/IPreparedModel.h>
30 #include <nnapi/Result.h>
31 #include <nnapi/TypeUtils.h>
32 #include <nnapi/Types.h>
33 #include <nnapi/hal/1.2/Conversions.h>
34 #include <nnapi/hal/1.2/ExecutionBurstController.h>
35 #include <nnapi/hal/1.2/ExecutionBurstUtils.h>
36 #include <nnapi/hal/CommonUtils.h>
37 #include <nnapi/hal/HandleError.h>
38 #include <nnapi/hal/ProtectCallback.h>
39
40 #include <memory>
41 #include <tuple>
42 #include <utility>
43 #include <vector>
44
45 // See hardware/interfaces/neuralnetworks/utils/README.md for more information on HIDL interface
46 // lifetimes across processes and for protecting asynchronous calls across HIDL.
47
48 namespace android::hardware::neuralnetworks::V1_3::utils {
49 namespace {
50
convertFencedExecutionCallbackResults(ErrorStatus status,const V1_2::Timing & timingLaunched,const V1_2::Timing & timingFenced)51 nn::GeneralResult<std::pair<nn::Timing, nn::Timing>> convertFencedExecutionCallbackResults(
52 ErrorStatus status, const V1_2::Timing& timingLaunched, const V1_2::Timing& timingFenced) {
53 HANDLE_HAL_STATUS(status) << "fenced execution callback info failed with " << toString(status);
54 return std::make_pair(NN_TRY(nn::convert(timingLaunched)), NN_TRY(nn::convert(timingFenced)));
55 }
56
fencedExecutionCallback(ErrorStatus status,const hidl_handle & syncFence,const sp<IFencedExecutionCallback> & callback)57 nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>> fencedExecutionCallback(
58 ErrorStatus status, const hidl_handle& syncFence,
59 const sp<IFencedExecutionCallback>& callback) {
60 HANDLE_HAL_STATUS(status) << "fenced execution failed with " << toString(status);
61
62 auto resultSyncFence = nn::SyncFence::createAsSignaled();
63 if (syncFence.getNativeHandle() != nullptr) {
64 auto sharedHandle = NN_TRY(nn::convert(syncFence));
65 resultSyncFence = NN_TRY(hal::utils::makeGeneralFailure(
66 nn::SyncFence::create(std::move(sharedHandle)), nn::ErrorStatus::GENERAL_FAILURE));
67 }
68
69 if (callback == nullptr) {
70 return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE) << "callback is null";
71 }
72
73 // Create callback which can be used to retrieve the execution error status and timings.
74 nn::ExecuteFencedInfoCallback resultCallback =
75 [callback]() -> nn::GeneralResult<std::pair<nn::Timing, nn::Timing>> {
76 auto cb = hal::utils::CallbackValue(convertFencedExecutionCallbackResults);
77
78 const auto ret = callback->getExecutionInfo(cb);
79 HANDLE_TRANSPORT_FAILURE(ret);
80
81 return cb.take();
82 };
83
84 return std::make_pair(std::move(resultSyncFence), std::move(resultCallback));
85 }
86
87 } // namespace
88
create(sp<V1_3::IPreparedModel> preparedModel,bool executeSynchronously)89 nn::GeneralResult<std::shared_ptr<const PreparedModel>> PreparedModel::create(
90 sp<V1_3::IPreparedModel> preparedModel, bool executeSynchronously) {
91 if (preparedModel == nullptr) {
92 return NN_ERROR() << "V1_3::utils::PreparedModel::create must have non-null preparedModel";
93 }
94
95 auto deathHandler = NN_TRY(hal::utils::DeathHandler::create(preparedModel));
96 return std::make_shared<const PreparedModel>(PrivateConstructorTag{}, executeSynchronously,
97 std::move(preparedModel), std::move(deathHandler));
98 }
99
PreparedModel(PrivateConstructorTag,bool executeSynchronously,sp<V1_3::IPreparedModel> preparedModel,hal::utils::DeathHandler deathHandler)100 PreparedModel::PreparedModel(PrivateConstructorTag /*tag*/, bool executeSynchronously,
101 sp<V1_3::IPreparedModel> preparedModel,
102 hal::utils::DeathHandler deathHandler)
103 : kExecuteSynchronously(executeSynchronously),
104 kPreparedModel(std::move(preparedModel)),
105 kDeathHandler(std::move(deathHandler)) {}
106
107 nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>
executeSynchronously(const Request & request,V1_2::MeasureTiming measure,const OptionalTimePoint & deadline,const OptionalTimeoutDuration & loopTimeoutDuration) const108 PreparedModel::executeSynchronously(const Request& request, V1_2::MeasureTiming measure,
109 const OptionalTimePoint& deadline,
110 const OptionalTimeoutDuration& loopTimeoutDuration) const {
111 auto cb = hal::utils::CallbackValue(executionCallback);
112
113 const auto ret = kPreparedModel->executeSynchronously_1_3(request, measure, deadline,
114 loopTimeoutDuration, cb);
115 HANDLE_TRANSPORT_FAILURE(ret);
116
117 return cb.take();
118 }
119
120 nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>
executeAsynchronously(const Request & request,V1_2::MeasureTiming measure,const OptionalTimePoint & deadline,const OptionalTimeoutDuration & loopTimeoutDuration) const121 PreparedModel::executeAsynchronously(const Request& request, V1_2::MeasureTiming measure,
122 const OptionalTimePoint& deadline,
123 const OptionalTimeoutDuration& loopTimeoutDuration) const {
124 const auto cb = sp<ExecutionCallback>::make();
125 const auto scoped = kDeathHandler.protectCallback(cb.get());
126
127 const auto ret =
128 kPreparedModel->execute_1_3(request, measure, deadline, loopTimeoutDuration, cb);
129 const auto status = HANDLE_TRANSPORT_FAILURE(ret);
130 if (status != ErrorStatus::OUTPUT_INSUFFICIENT_SIZE) {
131 HANDLE_HAL_STATUS(status) << "execution failed with " << toString(status);
132 }
133
134 return cb->get();
135 }
136
execute(const nn::Request & request,nn::MeasureTiming measure,const nn::OptionalTimePoint & deadline,const nn::OptionalDuration & loopTimeoutDuration) const137 nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>> PreparedModel::execute(
138 const nn::Request& request, nn::MeasureTiming measure,
139 const nn::OptionalTimePoint& deadline,
140 const nn::OptionalDuration& loopTimeoutDuration) const {
141 // Ensure that request is ready for IPC.
142 std::optional<nn::Request> maybeRequestInShared;
143 hal::utils::RequestRelocation relocation;
144 const nn::Request& requestInShared =
145 NN_TRY(hal::utils::makeExecutionFailure(hal::utils::convertRequestFromPointerToShared(
146 &request, nn::kDefaultRequestMemoryAlignment, nn::kMinMemoryPadding,
147 &maybeRequestInShared, &relocation)));
148
149 const auto hidlRequest = NN_TRY(hal::utils::makeExecutionFailure(convert(requestInShared)));
150 const auto hidlMeasure = NN_TRY(hal::utils::makeExecutionFailure(convert(measure)));
151 const auto hidlDeadline = NN_TRY(hal::utils::makeExecutionFailure(convert(deadline)));
152 const auto hidlLoopTimeoutDuration =
153 NN_TRY(hal::utils::makeExecutionFailure(convert(loopTimeoutDuration)));
154
155 return executeInternal(hidlRequest, hidlMeasure, hidlDeadline, hidlLoopTimeoutDuration,
156 relocation);
157 }
158
159 nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>
executeInternal(const Request & request,V1_2::MeasureTiming measure,const OptionalTimePoint & deadline,const OptionalTimeoutDuration & loopTimeoutDuration,const hal::utils::RequestRelocation & relocation) const160 PreparedModel::executeInternal(const Request& request, V1_2::MeasureTiming measure,
161 const OptionalTimePoint& deadline,
162 const OptionalTimeoutDuration& loopTimeoutDuration,
163 const hal::utils::RequestRelocation& relocation) const {
164 if (relocation.input) {
165 relocation.input->flush();
166 }
167
168 auto result = kExecuteSynchronously
169 ? executeSynchronously(request, measure, deadline, loopTimeoutDuration)
170 : executeAsynchronously(request, measure, deadline, loopTimeoutDuration);
171 auto [outputShapes, timing] = NN_TRY(std::move(result));
172
173 if (relocation.output) {
174 relocation.output->flush();
175 }
176 return std::make_pair(std::move(outputShapes), timing);
177 }
178
179 nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>>
executeFenced(const nn::Request & request,const std::vector<nn::SyncFence> & waitFor,nn::MeasureTiming measure,const nn::OptionalTimePoint & deadline,const nn::OptionalDuration & loopTimeoutDuration,const nn::OptionalDuration & timeoutDurationAfterFence) const180 PreparedModel::executeFenced(const nn::Request& request, const std::vector<nn::SyncFence>& waitFor,
181 nn::MeasureTiming measure, const nn::OptionalTimePoint& deadline,
182 const nn::OptionalDuration& loopTimeoutDuration,
183 const nn::OptionalDuration& timeoutDurationAfterFence) const {
184 // Ensure that request is ready for IPC.
185 std::optional<nn::Request> maybeRequestInShared;
186 hal::utils::RequestRelocation relocation;
187 const nn::Request& requestInShared = NN_TRY(hal::utils::convertRequestFromPointerToShared(
188 &request, nn::kDefaultRequestMemoryAlignment, nn::kMinMemoryPadding,
189 &maybeRequestInShared, &relocation));
190
191 const auto hidlRequest = NN_TRY(convert(requestInShared));
192 const auto hidlWaitFor = NN_TRY(hal::utils::convertSyncFences(waitFor));
193 const auto hidlMeasure = NN_TRY(convert(measure));
194 const auto hidlDeadline = NN_TRY(convert(deadline));
195 const auto hidlLoopTimeoutDuration = NN_TRY(convert(loopTimeoutDuration));
196 const auto hidlTimeoutDurationAfterFence = NN_TRY(convert(timeoutDurationAfterFence));
197
198 return executeFencedInternal(hidlRequest, hidlWaitFor, hidlMeasure, hidlDeadline,
199 hidlLoopTimeoutDuration, hidlTimeoutDurationAfterFence,
200 relocation);
201 }
202
203 nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>>
executeFencedInternal(const Request & request,const hidl_vec<hidl_handle> & waitFor,V1_2::MeasureTiming measure,const OptionalTimePoint & deadline,const OptionalTimeoutDuration & loopTimeoutDuration,const OptionalTimeoutDuration & timeoutDurationAfterFence,const hal::utils::RequestRelocation & relocation) const204 PreparedModel::executeFencedInternal(const Request& request, const hidl_vec<hidl_handle>& waitFor,
205 V1_2::MeasureTiming measure, const OptionalTimePoint& deadline,
206 const OptionalTimeoutDuration& loopTimeoutDuration,
207 const OptionalTimeoutDuration& timeoutDurationAfterFence,
208 const hal::utils::RequestRelocation& relocation) const {
209 if (relocation.input) {
210 relocation.input->flush();
211 }
212
213 auto cb = hal::utils::CallbackValue(fencedExecutionCallback);
214
215 const auto ret =
216 kPreparedModel->executeFenced(request, waitFor, measure, deadline, loopTimeoutDuration,
217 timeoutDurationAfterFence, cb);
218 HANDLE_TRANSPORT_FAILURE(ret);
219 auto [syncFence, callback] = NN_TRY(cb.take());
220
221 // If executeFenced required the request memory to be moved into shared memory, block here until
222 // the fenced execution has completed and flush the memory back.
223 if (relocation.output) {
224 const auto state = syncFence.syncWait({});
225 if (state != nn::SyncFence::FenceState::SIGNALED) {
226 return NN_ERROR() << "syncWait failed with " << state;
227 }
228 relocation.output->flush();
229 }
230
231 return std::make_pair(std::move(syncFence), std::move(callback));
232 }
233
createReusableExecution(const nn::Request & request,nn::MeasureTiming measure,const nn::OptionalDuration & loopTimeoutDuration) const234 nn::GeneralResult<nn::SharedExecution> PreparedModel::createReusableExecution(
235 const nn::Request& request, nn::MeasureTiming measure,
236 const nn::OptionalDuration& loopTimeoutDuration) const {
237 // Ensure that request is ready for IPC.
238 std::optional<nn::Request> maybeRequestInShared;
239 hal::utils::RequestRelocation relocation;
240 const nn::Request& requestInShared = NN_TRY(hal::utils::convertRequestFromPointerToShared(
241 &request, nn::kDefaultRequestMemoryAlignment, nn::kMinMemoryPadding,
242 &maybeRequestInShared, &relocation));
243
244 auto hidlRequest = NN_TRY(convert(requestInShared));
245 auto hidlMeasure = NN_TRY(convert(measure));
246 auto hidlLoopTimeoutDuration = NN_TRY(convert(loopTimeoutDuration));
247 return Execution::create(shared_from_this(), std::move(hidlRequest), std::move(relocation),
248 hidlMeasure, std::move(hidlLoopTimeoutDuration));
249 }
250
configureExecutionBurst() const251 nn::GeneralResult<nn::SharedBurst> PreparedModel::configureExecutionBurst() const {
252 auto self = shared_from_this();
253 auto fallback = [preparedModel = std::move(self)](
254 const nn::Request& request, nn::MeasureTiming measure,
255 const nn::OptionalTimePoint& deadline,
256 const nn::OptionalDuration& loopTimeoutDuration)
257 -> nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>> {
258 return preparedModel->execute(request, measure, deadline, loopTimeoutDuration);
259 };
260 const auto pollingTimeWindow = V1_2::utils::getBurstControllerPollingTimeWindow();
261 return V1_2::utils::ExecutionBurstController::create(shared_from_this(), kPreparedModel,
262 pollingTimeWindow);
263 }
264
getUnderlyingResource() const265 std::any PreparedModel::getUnderlyingResource() const {
266 sp<V1_3::IPreparedModel> resource = kPreparedModel;
267 return resource;
268 }
269
270 } // namespace android::hardware::neuralnetworks::V1_3::utils
271