1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_TYPES_NNAPI_IPREPARED_MODEL_H 18 #define ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_TYPES_NNAPI_IPREPARED_MODEL_H 19 20 #include <any> 21 #include <functional> 22 #include <memory> 23 #include <utility> 24 #include <vector> 25 26 #include "nnapi/Types.h" 27 28 namespace android::nn { 29 30 /** 31 * IPreparedModel describes a model that has been prepared for execution and is used to launch 32 * executions. 33 * 34 * This interface is thread-safe, and any class that implements this interface must be thread-safe. 35 */ 36 class IPreparedModel { 37 public: 38 /** 39 * Performs a synchronous execution on a prepared model. 40 * 41 * The execution is performed synchronously with respect to the caller. IPreparedModel::execute 42 * must verify the inputs to the function are correct. If there is an error, 43 * IPreparedModel::execute must immediately return {@link ErrorStatus::INVALID_ARGUMENT} as a 44 * ExecutionError. If the inputs to the function are valid and there is no error, 45 * IPreparedModel::execute must perform the execution, and must not return until the execution 46 * is complete. 47 * 48 * The caller must not change the content of any data object referenced by request (described by 49 * the {@link DataLocation} of a {@link RequestArgument}) until IPreparedModel::execute returns. 50 * IPreparedModel::execute must not change the content of any of the data objects corresponding 51 * to request inputs. 52 * 53 * If the prepared model was prepared from a model wherein all tensor operands have fully 54 * specified dimensions, and the inputs to the function are valid, and at execution time every 55 * operation's input operands have legal values, then the execution should complete 56 * successfully. There must be no failure unless the device itself is in a bad state. 57 * 58 * IPreparedModel::execute may be called with an optional deadline. If the execution is not 59 * able to be completed before the provided deadline, the execution may be aborted, and either 60 * {@link ErrorStatus::MISSED_DEADLINE_TRANSIENT} or {@link 61 * ErrorStatus::MISSED_DEADLINE_PERSISTENT} may be returned as a ExecutionError. 62 * 63 * @param request The input and output information on which the prepared model is to be 64 * executed. 65 * @param measure Specifies whether or not to measure duration of the execution. 66 * @param deadline Optional time point. If provided, execute is expected to complete by this 67 * time point. If it is not able to be completed by the deadline, the execution may be 68 * aborted. 69 * @param loopTimeoutDuration The maximum amount of time that should be spent executing a {@link 70 * OperationType::WHILE} operation. If a loop condition model does not output `false` within 71 * this duration, the execution must be aborted. If no loop timeout duration is provided, 72 * the maximum amount of time is {@link kControlFlowTimeoutDefault}. When provided, the 73 * duration must not exceed {@link kControlFlowTimeoutMaximum}. 74 * @param hints Specifies the optional device specific execution hints. The same token must not 75 * be repeated. It is allowed for the driver to ignore the user-provided hints. 76 * @param extensionNameToPrefix The mapping between extension names and prefixes of token 77 *. values. 78 * @return A pair consisting of: 79 * - A list of shape information of model output operands. The index into "outputShapes" 80 * corresponds to the index of the output operand in the Request outputs vector. 81 * outputShapes must be empty unless the execution is successful or the ExecutionResult is 82 * {@link ErrorStatus::OUTPUT_INSUFFICIENT_SIZE}. outputShapes may be empty if the 83 * execution is successful and all model output operands are fully-specified at execution 84 * time. outputShapes must have the same number of elements as the number of model output 85 * operands if the ExecutionResult is {@link ErrorStatus::OUTPUT_INSUFFICIENT_SIZE}, or if 86 * the execution is successful and the model has at least one output operand that is not 87 * fully-specified. 88 * - Duration of execution. Unless measure is YES and the execution is successful, all times 89 * must be reported as UINT64_MAX. A driver may choose to report any time as UINT64_MAX, 90 * indicating that measurement is not available. 91 */ 92 virtual ExecutionResult<std::pair<std::vector<OutputShape>, Timing>> execute( 93 const Request& request, MeasureTiming measure, const OptionalTimePoint& deadline, 94 const OptionalDuration& loopTimeoutDuration, 95 const std::vector<nn::TokenValuePair>& hints, 96 const std::vector<nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const = 0; 97 98 /** 99 * Launch a fenced asynchronous execution on a prepared model. 100 * 101 * The execution is performed asynchronously with respect to the caller. 102 * IPreparedModel::executeFenced must verify its inputs are correct, and the usages of memory 103 * pools allocated by IDevice::allocate are valid. If there is an error, 104 * IPreparedModel::executeFenced must immediately return {@link ErrorStatus::INVALID_ARGUMENT} 105 * as a GeneralError. If the inputs to the function are valid and there is no error, 106 * IPreparedModel::executeFenced must dispatch an asynchronous task to perform the execution in 107 * the background, and immediately return with a sync fence that will be signaled once the 108 * execution is completed and a callback that can be used by the client to query the duration 109 * and runtime error status. If the task has finished before the call returns, an empty handle 110 * may be returned for syncFence. The execution must wait for all the sync fences (if any) in 111 * waitFor to be signaled before starting the actual execution. 112 * 113 * When the asynchronous task has finished its execution, it must immediately signal the 114 * syncFence returned from the IPreparedModel::executeFenced call. After the syncFence is 115 * signaled, the task must not modify the content of any data object referenced by request 116 * (described by the {@link DataLocation} of a {@link Request::Argument}). 117 * 118 * IPreparedModel::executeFenced may be called with an optional deadline and an optional 119 * duration. If the execution is not able to be completed before the provided deadline or within 120 * the timeout duration (measured from when all sync fences in waitFor are signaled), whichever 121 * comes earlier, the execution may be aborted, and either {@link 122 * ErrorStatus::MISSED_DEADLINE_TRANSIENT} or {@link ErrorStatus::MISSED_DEADLINE_PERSISTENT} 123 * may be returned as an GeneralError. The error due to an abort must be sent the same way as 124 * other errors, described above. 125 * 126 * If any of the sync fences in waitFor changes to error status after the 127 * IPreparedModel::executeFenced call succeeds, or the execution is aborted because it cannot 128 * finish before the deadline has been reached or the duration has elapsed, the driver must 129 * immediately set the returned syncFence to error status. 130 * 131 * @param request The input and output information on which the prepared model is to be 132 * executed. 133 * @param waitFor A vector of sync fence file descriptors. The execution must wait for all sync 134 * fence to be signaled before starting the task. 135 * @param measure Specifies whether or not to measure duration of the execution. 136 * @param deadline The time by which execution is expected to complete. If the execution cannot 137 * be finished by the deadline, the execution may be aborted. 138 * @param loopTimeoutDuration The maximum amount of time that should be spent executing a {@link 139 * OperationType::WHILE} operation. If a loop condition model does not output `false` within 140 * this duration, the execution must be aborted. If no loop timeout duration is provided, 141 * the maximum amount of time is {@link kControlFlowTimeoutDefault}. When provided, the 142 * duration must not exceed {@link kControlFlowTimeoutMaximum}. 143 * @param timeoutDurationAfterFence The timeout duration within which the execution is expected 144 * to complete after all sync fences in waitFor are signaled. 145 * @param hints Specifies the optional device specific execution hints. The same token must not 146 * be repeated. It is allowed for the driver to ignore the user-provided hints. 147 * @param extensionNameToPrefix The mapping between extension names and prefixes of token 148 *. values. 149 * @return A pair consisting of: 150 * - A syncFence that will be triggered when the task is completed. The syncFence will be 151 * set to error if critical error occurs when doing actual evaluation. 152 * - A callback can be used to query information like duration and detailed runtime error 153 * status when the task is completed. 154 */ 155 virtual GeneralResult<std::pair<SyncFence, ExecuteFencedInfoCallback>> executeFenced( 156 const Request& request, const std::vector<SyncFence>& waitFor, MeasureTiming measure, 157 const OptionalTimePoint& deadline, const OptionalDuration& loopTimeoutDuration, 158 const OptionalDuration& timeoutDurationAfterFence, 159 const std::vector<nn::TokenValuePair>& hints, 160 const std::vector<nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const = 0; 161 162 /** 163 * Create a reusable execution from a request and execution configurations. 164 * 165 * IPreparedModel::createReusableExecution must verify the inputs to the function are correct. 166 * If there is an error, IPreparedModel::createReusableExecution must immediately return 167 * {@link ErrorStatus::INVALID_ARGUMENT} as a GeneralError. If the inputs to the function are 168 * valid and there is no error, IPreparedModel::createReusableExecution must construct a 169 * reusable execution. 170 * 171 * @param request The input and output information on which the prepared model is to be 172 * executed. 173 * @param measure Specifies whether or not to measure duration of the computations performed 174 * with the returned execution. 175 * @param loopTimeoutDuration The maximum amount of time that should be spent executing a {@link 176 * OperationType::WHILE} operation. During a computation with the returned execution, if a 177 * loop condition model does not output `false` within this duration, the computation must 178 * be aborted. If no loop timeout duration is provided, the maximum amount of time is 179 * {@link LoopTimeoutDurationNs::DEFAULT}. When provided, the duration must not exceed 180 * {@link LoopTimeoutDurationNs::MAXIMUM}. 181 * @param hints Specifies the optional device specific execution hints. It is allowed for the 182 * driver to ignore the user-provided hints. 183 * @param extensionNameToPrefix The mapping between extension names and prefixes of token 184 *. values. 185 * @return execution An IExecution object representing a reusable execution that has been 186 * specialized for a fixed request, otherwise GeneralError. 187 */ 188 virtual GeneralResult<SharedExecution> createReusableExecution( 189 const Request& request, MeasureTiming measure, 190 const OptionalDuration& loopTimeoutDuration, 191 const std::vector<nn::TokenValuePair>& hints, 192 const std::vector<nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const = 0; 193 194 /** 195 * Creates a burst controller on a prepared model. 196 * 197 * @return ExecutionBurstController Execution burst controller object, otherwise GeneralError. 198 */ 199 virtual GeneralResult<SharedBurst> configureExecutionBurst() const = 0; 200 201 /** 202 * Return the resource that the IPreparedModel wraps, or any empty std::any. 203 * 204 * This method is used for IDevice::allocate. 205 * 206 * @return std::any containing the underlying resource. 207 */ 208 virtual std::any getUnderlyingResource() const = 0; 209 210 // Public virtual destructor to allow objects to be stored (and destroyed) as smart pointers. 211 // E.g., std::unique_ptr<IPreparedModel>. 212 virtual ~IPreparedModel() = default; 213 214 protected: 215 // Protect the non-destructor special member functions to prevent object slicing. 216 IPreparedModel() = default; 217 IPreparedModel(const IPreparedModel&) = default; 218 IPreparedModel(IPreparedModel&&) noexcept = default; 219 IPreparedModel& operator=(const IPreparedModel&) = default; 220 IPreparedModel& operator=(IPreparedModel&&) noexcept = default; 221 }; 222 223 } // namespace android::nn 224 225 #endif // ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_TYPES_NNAPI_IPREPARED_MODEL_H 226