1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "ExecutionBurstServer"
18
19 #include "ExecutionBurstServer.h"
20 #include "Conversions.h"
21 #include "ExecutionBurstUtils.h"
22
23 #include <android-base/logging.h>
24 #include <nnapi/IBurst.h>
25 #include <nnapi/Result.h>
26 #include <nnapi/TypeUtils.h>
27 #include <nnapi/Types.h>
28 #include <nnapi/Validation.h>
29 #include <nnapi/hal/1.0/Conversions.h>
30 #include <nnapi/hal/HandleError.h>
31 #include <nnapi/hal/ProtectCallback.h>
32 #include <nnapi/hal/TransferValue.h>
33
34 #include <algorithm>
35 #include <cstring>
36 #include <limits>
37 #include <map>
38 #include <memory>
39 #include <tuple>
40 #include <utility>
41 #include <vector>
42
43 #include "Tracing.h"
44
45 namespace android::hardware::neuralnetworks::V1_2::utils {
46 namespace {
47
48 using neuralnetworks::utils::makeExecutionFailure;
49
50 constexpr V1_2::Timing kNoTiming = {std::numeric_limits<uint64_t>::max(),
51 std::numeric_limits<uint64_t>::max()};
52
getMemoriesCallback(V1_0::ErrorStatus status,const hidl_vec<hidl_memory> & memories)53 nn::GeneralResult<std::vector<nn::SharedMemory>> getMemoriesCallback(
54 V1_0::ErrorStatus status, const hidl_vec<hidl_memory>& memories) {
55 HANDLE_HAL_STATUS(status) << "getting burst memories failed with " << toString(status);
56 std::vector<nn::SharedMemory> canonicalMemories;
57 canonicalMemories.reserve(memories.size());
58 for (const auto& memory : memories) {
59 canonicalMemories.push_back(NN_TRY(nn::convert(memory)));
60 }
61 return canonicalMemories;
62 }
63
64 } // anonymous namespace
65
MemoryCache(nn::SharedBurst burstExecutor,sp<IBurstCallback> burstCallback)66 ExecutionBurstServer::MemoryCache::MemoryCache(nn::SharedBurst burstExecutor,
67 sp<IBurstCallback> burstCallback)
68 : kBurstExecutor(std::move(burstExecutor)), kBurstCallback(std::move(burstCallback)) {
69 CHECK(kBurstExecutor != nullptr);
70 CHECK(kBurstCallback != nullptr);
71 }
72
73 nn::GeneralResult<std::vector<std::pair<nn::SharedMemory, nn::IBurst::OptionalCacheHold>>>
getCacheEntries(const std::vector<int32_t> & slots)74 ExecutionBurstServer::MemoryCache::getCacheEntries(const std::vector<int32_t>& slots) {
75 std::lock_guard guard(mMutex);
76 NN_TRY(ensureCacheEntriesArePresentLocked(slots));
77
78 std::vector<std::pair<nn::SharedMemory, nn::IBurst::OptionalCacheHold>> results;
79 results.reserve(slots.size());
80 for (int32_t slot : slots) {
81 results.push_back(NN_TRY(getCacheEntryLocked(slot)));
82 }
83
84 return results;
85 }
86
ensureCacheEntriesArePresentLocked(const std::vector<int32_t> & slots)87 nn::GeneralResult<void> ExecutionBurstServer::MemoryCache::ensureCacheEntriesArePresentLocked(
88 const std::vector<int32_t>& slots) {
89 const auto slotIsKnown = [this](int32_t slot)
90 REQUIRES(mMutex) { return mCache.count(slot) > 0; };
91
92 // find unique unknown slots
93 std::vector<int32_t> unknownSlots = slots;
94 std::sort(unknownSlots.begin(), unknownSlots.end());
95 auto unknownSlotsEnd = std::unique(unknownSlots.begin(), unknownSlots.end());
96 unknownSlotsEnd = std::remove_if(unknownSlots.begin(), unknownSlotsEnd, slotIsKnown);
97 unknownSlots.erase(unknownSlotsEnd, unknownSlots.end());
98
99 // quick-exit if all slots are known
100 if (unknownSlots.empty()) {
101 return {};
102 }
103
104 auto cb = neuralnetworks::utils::CallbackValue(getMemoriesCallback);
105
106 const auto ret = kBurstCallback->getMemories(unknownSlots, cb);
107 HANDLE_TRANSPORT_FAILURE(ret);
108
109 auto returnedMemories = NN_TRY(cb.take());
110
111 if (returnedMemories.size() != unknownSlots.size()) {
112 return NN_ERROR()
113 << "ExecutionBurstServer::MemoryCache::ensureCacheEntriesArePresentLocked: Error "
114 "retrieving memories -- count mismatch between requested memories ("
115 << unknownSlots.size() << ") and returned memories (" << returnedMemories.size()
116 << ")";
117 }
118
119 // add memories to unknown slots
120 for (size_t i = 0; i < unknownSlots.size(); ++i) {
121 addCacheEntryLocked(unknownSlots[i], std::move(returnedMemories[i]));
122 }
123
124 return {};
125 }
126
127 nn::GeneralResult<std::pair<nn::SharedMemory, nn::IBurst::OptionalCacheHold>>
getCacheEntryLocked(int32_t slot)128 ExecutionBurstServer::MemoryCache::getCacheEntryLocked(int32_t slot) {
129 if (const auto iter = mCache.find(slot); iter != mCache.end()) {
130 return iter->second;
131 }
132 return NN_ERROR()
133 << "ExecutionBurstServer::MemoryCache::getCacheEntryLocked failed because slot " << slot
134 << " is not present in the cache";
135 }
136
addCacheEntryLocked(int32_t slot,nn::SharedMemory memory)137 void ExecutionBurstServer::MemoryCache::addCacheEntryLocked(int32_t slot, nn::SharedMemory memory) {
138 auto hold = kBurstExecutor->cacheMemory(memory);
139 mCache.emplace(slot, std::make_pair(std::move(memory), std::move(hold)));
140 }
141
removeCacheEntry(int32_t slot)142 void ExecutionBurstServer::MemoryCache::removeCacheEntry(int32_t slot) {
143 std::lock_guard guard(mMutex);
144 mCache.erase(slot);
145 }
146
147 // ExecutionBurstServer methods
148
create(const sp<IBurstCallback> & callback,const MQDescriptorSync<FmqRequestDatum> & requestChannel,const MQDescriptorSync<FmqResultDatum> & resultChannel,nn::SharedBurst burstExecutor,std::chrono::microseconds pollingTimeWindow)149 nn::GeneralResult<sp<ExecutionBurstServer>> ExecutionBurstServer::create(
150 const sp<IBurstCallback>& callback, const MQDescriptorSync<FmqRequestDatum>& requestChannel,
151 const MQDescriptorSync<FmqResultDatum>& resultChannel, nn::SharedBurst burstExecutor,
152 std::chrono::microseconds pollingTimeWindow) {
153 // check inputs
154 if (callback == nullptr || burstExecutor == nullptr) {
155 return NN_ERROR() << "ExecutionBurstServer::create passed a nullptr";
156 }
157
158 // create FMQ objects
159 auto requestChannelReceiver =
160 NN_TRY(RequestChannelReceiver::create(requestChannel, pollingTimeWindow));
161 auto resultChannelSender = NN_TRY(ResultChannelSender::create(resultChannel));
162
163 // check FMQ objects
164 CHECK(requestChannelReceiver != nullptr);
165 CHECK(resultChannelSender != nullptr);
166
167 // make and return context
168 return sp<ExecutionBurstServer>::make(PrivateConstructorTag{}, callback,
169 std::move(requestChannelReceiver),
170 std::move(resultChannelSender), std::move(burstExecutor));
171 }
172
ExecutionBurstServer(PrivateConstructorTag,const sp<IBurstCallback> & callback,std::unique_ptr<RequestChannelReceiver> requestChannel,std::unique_ptr<ResultChannelSender> resultChannel,nn::SharedBurst burstExecutor)173 ExecutionBurstServer::ExecutionBurstServer(PrivateConstructorTag /*tag*/,
174 const sp<IBurstCallback>& callback,
175 std::unique_ptr<RequestChannelReceiver> requestChannel,
176 std::unique_ptr<ResultChannelSender> resultChannel,
177 nn::SharedBurst burstExecutor)
178 : mCallback(callback),
179 mRequestChannelReceiver(std::move(requestChannel)),
180 mResultChannelSender(std::move(resultChannel)),
181 mBurstExecutor(std::move(burstExecutor)),
182 mMemoryCache(mBurstExecutor, mCallback) {
183 // TODO: highly document the threading behavior of this class
184 mWorker = std::thread([this] { task(); });
185 }
186
~ExecutionBurstServer()187 ExecutionBurstServer::~ExecutionBurstServer() {
188 // set teardown flag
189 mTeardown = true;
190 mRequestChannelReceiver->invalidate();
191
192 // wait for task thread to end
193 mWorker.join();
194 }
195
freeMemory(int32_t slot)196 Return<void> ExecutionBurstServer::freeMemory(int32_t slot) {
197 mMemoryCache.removeCacheEntry(slot);
198 return Void();
199 }
200
task()201 void ExecutionBurstServer::task() {
202 // loop until the burst object is being destroyed
203 while (!mTeardown) {
204 // receive request
205 auto arguments = mRequestChannelReceiver->getBlocking();
206
207 // if the request packet was not properly received, return a generic error and skip the
208 // execution
209 //
210 // if the burst is being torn down, skip the execution so the "task" function can end
211 if (!arguments.has_value()) {
212 if (!mTeardown) {
213 mResultChannelSender->send(V1_0::ErrorStatus::GENERAL_FAILURE, {}, kNoTiming);
214 }
215 continue;
216 }
217
218 // unpack the arguments; types are Request, std::vector<int32_t>, and MeasureTiming,
219 // respectively
220 const auto [requestWithoutPools, slotsOfPools, measure] = std::move(arguments).value();
221
222 auto result = execute(requestWithoutPools, slotsOfPools, measure);
223
224 // return result
225 if (result.has_value()) {
226 const auto& [outputShapes, timing] = result.value();
227 mResultChannelSender->send(V1_0::ErrorStatus::NONE, outputShapes, timing);
228 } else {
229 const auto& [message, code, outputShapes] = result.error();
230 LOG(ERROR) << "IBurst::execute failed with " << code << ": " << message;
231 mResultChannelSender->send(convert(code).value(), convert(outputShapes).value(),
232 kNoTiming);
233 }
234 }
235 }
236
execute(const V1_0::Request & requestWithoutPools,const std::vector<int32_t> & slotsOfPools,MeasureTiming measure)237 nn::ExecutionResult<std::pair<hidl_vec<OutputShape>, Timing>> ExecutionBurstServer::execute(
238 const V1_0::Request& requestWithoutPools, const std::vector<int32_t>& slotsOfPools,
239 MeasureTiming measure) {
240 NNTRACE_FULL(NNTRACE_LAYER_IPC, NNTRACE_PHASE_EXECUTION,
241 "ExecutionBurstServer getting memory, executing, and returning results");
242
243 // ensure executor with cache has required memory
244 const auto cacheEntries =
245 NN_TRY(makeExecutionFailure(mMemoryCache.getCacheEntries(slotsOfPools)));
246
247 // convert request, populating its pools
248 // This code performs an unvalidated convert because the request object without its pools is
249 // invalid because it is incomplete. Instead, the validation is performed after the memory pools
250 // have been added to the request.
251 auto canonicalRequest =
252 NN_TRY(makeExecutionFailure(nn::unvalidatedConvert(requestWithoutPools)));
253 CHECK(canonicalRequest.pools.empty());
254 std::transform(cacheEntries.begin(), cacheEntries.end(),
255 std::back_inserter(canonicalRequest.pools),
256 [](const auto& cacheEntry) { return cacheEntry.first; });
257 NN_TRY(makeExecutionFailure(validate(canonicalRequest)));
258
259 nn::MeasureTiming canonicalMeasure = NN_TRY(makeExecutionFailure(nn::convert(measure)));
260
261 const auto [outputShapes, timing] =
262 NN_TRY(mBurstExecutor->execute(canonicalRequest, canonicalMeasure, {}, {}));
263
264 return std::make_pair(NN_TRY(makeExecutionFailure(convert(outputShapes))),
265 NN_TRY(makeExecutionFailure(convert(timing))));
266 }
267
268 } // namespace android::hardware::neuralnetworks::V1_2::utils
269