1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_TYPES_NNAPI_IBURST_H 18 #define ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_TYPES_NNAPI_IBURST_H 19 20 #include <android-base/scopeguard.h> 21 22 #include <functional> 23 #include <memory> 24 #include <optional> 25 #include <utility> 26 #include <vector> 27 28 #include "nnapi/Types.h" 29 30 namespace android::nn { 31 32 /** 33 * IBurst represents a burst execution object. 34 * 35 * Burst executions are a sequence of executions of the same prepared model that occur in rapid 36 * succession, such as frames of a camera capture or successive audio samples. A burst object is 37 * used to control a set of burst executions, and to preserve resources between executions, enabling 38 * executions to have lower overhead. Burst objects enable some optimizations: 39 * (1) A burst object is created before a sequence of executions, and freed when the sequence has 40 * ended. Because of this, the lifetime of the burst object hints to a driver how long it should 41 * remain in a high performance state. 42 * (2) A burst object can preserve resources between executions. For example, a driver can map a 43 * memory object on the first execution and cache the mapping in the burst object for reuse in 44 * subsequent executions. Any cached resource can be released when the burst object is destroyed 45 * or when the NNAPI runtime notifies the burst object that the resource is no longer required. 46 * (3) A burst object may be used for at most one execution at a time. This enables any transient 47 * execution resources such as intermediate tensors to be allocated once when the burst object 48 * is created and freed when the burst object is destroyed. 49 * 50 * This interface is thread-safe, and any class that implements this interface must be thread-safe. 51 */ 52 class IBurst { 53 public: 54 using OptionalCacheHold = std::shared_ptr<const base::ScopeGuard<std::function<void()>>>; 55 56 /** 57 * Cache a memory object in the burst. 58 * 59 * This can enable multiple executions that reuse the same memory to be more efficient. 60 * 61 * @param memory The memory object to be cached as long as CacheHandle is held. 62 * @return An optional cache handle that will release the corresponding cahced object once the 63 * cache handle is released, or nullptr. 64 */ 65 virtual OptionalCacheHold cacheMemory(const SharedMemory& memory) const = 0; 66 67 /** 68 * Performs a synchronous execution on a prepared model. 69 * 70 * At most one execution may occur on a burst object at any given time. 71 * 72 * The execution is performed synchronously with respect to the caller. IBurst::execute must 73 * verify the inputs to the function are correct. If there is an error, IBurst::execute must 74 * immediately return {@link ErrorStatus::INVALID_ARGUMENT} as a ExecutionError. If the inputs 75 * to the function are valid and there is no error, IBurst::execute must perform the execution, 76 * and must not return until the execution is complete. 77 * 78 * The caller must not change the content of any data object referenced by request (described by 79 * the {@link DataLocation} of a {@link RequestArgument}) until IBurst::execute returns. 80 * IBurst::execute must not change the content of any of the data objects corresponding to 81 * request inputs. 82 * 83 * If the prepared model was prepared from a model wherein all tensor operands have fully 84 * specified dimensions, and the inputs to the function are valid, and at execution time every 85 * operation's input operands have legal values, then the execution should complete 86 * successfully. There must be no failure unless the device itself is in a bad state. 87 * 88 * @param request The input and output information on which the prepared model is to be 89 * executed. 90 * @param measure Specifies whether or not to measure duration of the execution. 91 * @param deadline Optional time point. If provided, execute is expected to complete by this 92 * time point. If it is not able to be completed by the deadline, the execution may be 93 * aborted. 94 * @param loopTimeoutDuration The maximum amount of time that should be spent executing a {@link 95 * OperationType::WHILE} operation. If a loop condition model does not output `false` within 96 * this duration, the execution must be aborted. If no loop timeout duration is provided, 97 * the maximum amount of time is {@link kControlFlowTimeoutDefault}. When provided, the 98 * duration must not exceed {@link kControlFlowTimeoutMaximum}. 99 * @param hints Specifies the optional device specific execution hints. It is allowed for the 100 * driver to ignore the user-provided hints. 101 * @param extensionNameToPrefix The mapping between extension names and prefixes of token 102 *. values. 103 * @return A pair consisting of: 104 * - A list of shape information of model output operands. The index into "outputShapes" 105 * corresponds to the index of the output operand in the Request outputs vector. 106 * outputShapes must be empty unless the execution is successful or the ExecutionResult is 107 * {@link ErrorStatus::OUTPUT_INSUFFICIENT_SIZE}. outputShapes may be empty if the 108 * execution is successful and all model output operands are fully-specified at execution 109 * time. outputShapes must have the same number of elements as the number of model output 110 * operands if the ExecutionResult is {@link ErrorStatus::OUTPUT_INSUFFICIENT_SIZE}, or if 111 * the execution is successful and the model has at least one output operand that is not 112 * fully-specified. 113 * - Duration of execution. Unless measure is YES and the execution is successful, all times 114 * must be reported as std::nullopt. A driver may choose to report any time as 115 * std::nullopt, indicating that measurement is not available. 116 */ 117 virtual ExecutionResult<std::pair<std::vector<OutputShape>, Timing>> execute( 118 const Request& request, MeasureTiming measure, const nn::OptionalTimePoint& deadline, 119 const nn::OptionalDuration& loopTimeoutDuration, 120 const std::vector<nn::TokenValuePair>& hints, 121 const std::vector<nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const = 0; 122 123 /** 124 * Create a reusable burst execution object. 125 * 126 * IBurst::createReusableExecution must verify the inputs to the function are correct. If there 127 * is an error, IBurst::createReusableExecution must immediately return {@link 128 * ErrorStatus::INVALID_ARGUMENT} as a GeneralError. If the inputs to the function are valid and 129 * there is no error, IBurst::createReusableExecution must construct a reusable execution. 130 * 131 * @param request The input and output information on which the prepared model is to be 132 * executed. 133 * @param measure Specifies whether or not to measure duration of the computation. 134 * @param loopTimeoutDuration The maximum amount of time that should be spent executing a {@link 135 * OperationType::WHILE} operation. If a loop condition model does not output `false` within 136 * this duration, the execution must be aborted. If no loop timeout duration is provided, 137 * the maximum amount of time is {@link kControlFlowTimeoutDefault}. When provided, the 138 * duration must not exceed {@link kControlFlowTimeoutMaximum}. 139 * @param hints Specifies the optional device specific execution hints. The same token must not 140 * be repeated. It is allowed for the driver to ignore the user-provided hints. 141 * @param extensionNameToPrefix The mapping between extension names and prefixes of token 142 *. values. 143 * @return execution An IExecution object representing a reusable burst execution that has been 144 * specialized for a fixed request, otherwise GeneralError. 145 */ 146 virtual GeneralResult<SharedExecution> createReusableExecution( 147 const Request& request, MeasureTiming measure, 148 const nn::OptionalDuration& loopTimeoutDuration, 149 const std::vector<nn::TokenValuePair>& hints, 150 const std::vector<nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const = 0; 151 152 // Public virtual destructor to allow objects to be stored (and destroyed) as smart pointers. 153 // E.g., std::unique_ptr<IBurst>. 154 virtual ~IBurst() = default; 155 156 protected: 157 // Protect the non-destructor special member functions to prevent object slicing. 158 IBurst() = default; 159 IBurst(const IBurst&) = default; 160 IBurst(IBurst&&) noexcept = default; 161 IBurst& operator=(const IBurst&) = default; 162 IBurst& operator=(IBurst&&) noexcept = default; 163 }; 164 165 } // namespace android::nn 166 167 #endif // ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_TYPES_NNAPI_IBURST_H 168