1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "EvsStateControl.h"
17
18 #include "FormatConvert.h"
19 #include "RenderDirectView.h"
20 #include "RenderPixelCopy.h"
21 #include "RenderTopView.h"
22
23 #include <aidl/android/hardware/automotive/evs/CameraDesc.h>
24 #include <aidl/android/hardware/automotive/evs/DisplayState.h>
25 #include <aidl/android/hardware/automotive/evs/IEvsDisplay.h>
26 #include <aidl/android/hardware/automotive/evs/IEvsEnumerator.h>
27 #include <aidl/android/hardware/automotive/vehicle/VehicleGear.h>
28 #include <aidl/android/hardware/automotive/vehicle/VehicleProperty.h>
29 #include <aidl/android/hardware/automotive/vehicle/VehiclePropertyType.h>
30 #include <aidl/android/hardware/automotive/vehicle/VehicleTurnSignal.h>
31 #include <android-base/logging.h>
32 #include <android/binder_manager.h>
33 #include <utils/SystemClock.h>
34
35 #include <inttypes.h>
36 #include <stdio.h>
37 #include <string.h>
38
39 namespace {
40
41 using aidl::android::hardware::automotive::evs::BufferDesc;
42 using aidl::android::hardware::automotive::evs::CameraDesc;
43 using aidl::android::hardware::automotive::evs::DisplayState;
44 using aidl::android::hardware::automotive::evs::IEvsDisplay;
45 using aidl::android::hardware::automotive::evs::IEvsEnumerator;
46 using aidl::android::hardware::automotive::vehicle::VehicleGear;
47 using aidl::android::hardware::automotive::vehicle::VehicleProperty;
48 using aidl::android::hardware::automotive::vehicle::VehiclePropertyType;
49 using aidl::android::hardware::automotive::vehicle::VehiclePropValue;
50 using aidl::android::hardware::automotive::vehicle::VehicleTurnSignal;
51 using android::frameworks::automotive::vhal::ErrorCode;
52 using android::frameworks::automotive::vhal::IHalPropValue;
53 using android::frameworks::automotive::vhal::IVhalClient;
54 using android::frameworks::automotive::vhal::VhalClientResult;
55
isSfReady()56 bool isSfReady() {
57 return ndk::SpAIBinder(AServiceManager_checkService("SurfaceFlinger")).get() != nullptr;
58 }
59
getPropType(VehicleProperty prop)60 inline constexpr VehiclePropertyType getPropType(VehicleProperty prop) {
61 return static_cast<VehiclePropertyType>(static_cast<int32_t>(prop) &
62 static_cast<int32_t>(VehiclePropertyType::MASK));
63 }
64
65 } // namespace
66
EvsStateControl(std::shared_ptr<IVhalClient> pVnet,std::shared_ptr<IEvsEnumerator> pEvs,const std::shared_ptr<IEvsDisplay> & pDisplay,const ConfigManager & config)67 EvsStateControl::EvsStateControl(std::shared_ptr<IVhalClient> pVnet,
68 std::shared_ptr<IEvsEnumerator> pEvs,
69 const std::shared_ptr<IEvsDisplay>& pDisplay,
70 const ConfigManager& config) :
71 mVehicle(pVnet),
72 mEvs(pEvs),
73 mDisplay(pDisplay),
74 mConfig(config),
75 mCurrentState(OFF),
76 mEvsStats(EvsStats::build()) {
77 // Initialize the property value containers we'll be updating (they'll be zeroed by default)
78 static_assert(getPropType(VehicleProperty::GEAR_SELECTION) == VehiclePropertyType::INT32,
79 "Unexpected type for GEAR_SELECTION property");
80 static_assert(getPropType(VehicleProperty::TURN_SIGNAL_STATE) == VehiclePropertyType::INT32,
81 "Unexpected type for TURN_SIGNAL_STATE property");
82
83 mGearValue.prop = static_cast<int32_t>(VehicleProperty::GEAR_SELECTION);
84 mTurnSignalValue.prop = static_cast<int32_t>(VehicleProperty::TURN_SIGNAL_STATE);
85
86 // This way we only ever deal with cameras which exist in the system
87 // Build our set of cameras for the states we support
88 LOG(DEBUG) << "Requesting camera list";
89 std::vector<CameraDesc> cameraList;
90 if (auto status = mEvs->getCameraList(&cameraList); !status.isOk()) {
91 LOG(ERROR) << "Failed to get the camera list.";
92 return;
93 }
94
95 LOG(INFO) << "Camera list callback received " << cameraList.size() << "cameras.";
96 for (auto&& cam : cameraList) {
97 LOG(DEBUG) << "Found camera " << cam.id;
98 bool cameraConfigFound = false;
99
100 // Check our configuration for information about this camera
101 // Note that a camera can have a compound function string
102 // such that a camera can be "right/reverse" and be used for both.
103 // If more than one camera is listed for a given function, we'll
104 // list all of them and let the UX/rendering logic use one, some
105 // or all of them as appropriate.
106 for (auto&& info : config.getCameras()) {
107 if (cam.id == info.cameraId) {
108 // We found a match!
109 if (info.function.find("reverse") != std::string::npos) {
110 mCameraList[State::REVERSE].emplace_back(info);
111 mCameraDescList[State::REVERSE].emplace_back(cam);
112 }
113 if (info.function.find("right") != std::string::npos) {
114 mCameraList[State::RIGHT].emplace_back(info);
115 mCameraDescList[State::RIGHT].emplace_back(cam);
116 }
117 if (info.function.find("left") != std::string::npos) {
118 mCameraList[State::LEFT].emplace_back(info);
119 mCameraDescList[State::LEFT].emplace_back(cam);
120 }
121 if (info.function.find("park") != std::string::npos) {
122 mCameraList[State::PARKING].emplace_back(info);
123 mCameraDescList[State::PARKING].emplace_back(cam);
124 }
125 cameraConfigFound = true;
126 break;
127 }
128 }
129 if (!cameraConfigFound) {
130 LOG(WARNING) << "No config information for hardware camera " << cam.id;
131 }
132 }
133
134 LOG(INFO) << "State controller ready";
135 }
136
startUpdateLoop()137 bool EvsStateControl::startUpdateLoop() {
138 // Create the thread and report success if it gets started
139 mRenderThread = std::thread([this]() { updateLoop(); });
140 return mRenderThread.joinable();
141 }
142
terminateUpdateLoop()143 void EvsStateControl::terminateUpdateLoop() {
144 if (mRenderThread.get_id() == std::this_thread::get_id()) {
145 // We should not join by ourselves
146 mRenderThread.detach();
147 } else if (mRenderThread.joinable()) {
148 // Join a rendering thread
149 mRenderThread.join();
150 }
151 }
152
postCommand(const Command & cmd,bool clear)153 void EvsStateControl::postCommand(const Command& cmd, bool clear) {
154 // Push the command onto the queue watched by updateLoop
155 mLock.lock();
156 if (clear) {
157 std::queue<Command> emptyQueue;
158 std::swap(emptyQueue, mCommandQueue);
159 }
160
161 mCommandQueue.push(cmd);
162 mLock.unlock();
163
164 // Send a signal to wake updateLoop in case it is asleep
165 mWakeSignal.notify_all();
166 }
167
updateLoop()168 void EvsStateControl::updateLoop() {
169 LOG(DEBUG) << "Starting EvsStateControl update loop";
170
171 bool run = true;
172 while (run) {
173 // Process incoming commands
174 std::shared_ptr<IEvsDisplay> displayHandle;
175 {
176 std::lock_guard lock(mLock);
177 while (!mCommandQueue.empty()) {
178 const Command& cmd = mCommandQueue.front();
179 switch (cmd.operation) {
180 case Op::EXIT:
181 run = false;
182 break;
183 case Op::CHECK_VEHICLE_STATE:
184 // Just running selectStateForCurrentConditions below will take care of this
185 break;
186 case Op::TOUCH_EVENT:
187 // Implement this given the x/y location of the touch event
188 break;
189 }
190 mCommandQueue.pop();
191 }
192
193 displayHandle = mDisplay.lock();
194 }
195
196 if (!displayHandle) {
197 LOG(ERROR) << "We've lost the display";
198 break;
199 }
200
201 // Review vehicle state and choose an appropriate renderer
202 if (!selectStateForCurrentConditions()) {
203 LOG(ERROR) << "selectStateForCurrentConditions failed so we're going to die";
204 break;
205 }
206
207 // If we have an active renderer, give it a chance to draw
208 if (mCurrentRenderer) {
209 // Get the output buffer we'll use to display the imagery
210 BufferDesc tgtBuffer;
211 if (auto status = displayHandle->getTargetBuffer(&tgtBuffer); !status.isOk()) {
212 LOG(ERROR) << "Didn't get requested output buffer -- skipping this frame.";
213 run = false;
214 } else {
215 // Generate our output image
216 if (!mCurrentRenderer->drawFrame(tgtBuffer)) {
217 // If drawing failed, we want to exit quickly so an app restart can happen
218 run = false;
219 }
220
221 // Send the finished image back for display
222 displayHandle->returnTargetBufferForDisplay(tgtBuffer);
223
224 if (!mFirstFrameIsDisplayed) {
225 mFirstFrameIsDisplayed = true;
226 // returnTargetBufferForDisplay() is finished, the frame should be displayed
227 mEvsStats.finishComputingFirstFrameLatency(android::uptimeMillis());
228 }
229 }
230 } else if (run) {
231 // No active renderer, so sleep until somebody wakes us with another command
232 // or exit if we received EXIT command
233 std::unique_lock<std::mutex> lock(mLock);
234 mWakeSignal.wait(lock);
235 }
236 }
237
238 LOG(WARNING) << "EvsStateControl update loop ending";
239
240 if (mCurrentRenderer) {
241 // Deactive the renderer
242 mCurrentRenderer->deactivate();
243 }
244
245 // If `ICarTelemetry` service was not ready before, we need to try sending data again.
246 mEvsStats.sendCollectedDataBlocking();
247
248 printf("Shutting down app due to state control loop ending\n");
249 LOG(ERROR) << "Shutting down app due to state control loop ending";
250 }
251
selectStateForCurrentConditions()252 bool EvsStateControl::selectStateForCurrentConditions() {
253 static int32_t sMockGear = mConfig.getMockGearSignal();
254 static int32_t sMockSignal = int32_t(VehicleTurnSignal::NONE);
255
256 if (mVehicle != nullptr) {
257 // Query the car state
258 if (invokeGet(&mGearValue) != ErrorCode::OK) {
259 LOG(ERROR) << "GEAR_SELECTION not available from vehicle. Exiting.";
260 return false;
261 }
262 if ((mTurnSignalValue.prop == 0) || (invokeGet(&mTurnSignalValue) != ErrorCode::OK)) {
263 // Silently treat missing turn signal state as no turn signal active
264 mTurnSignalValue.value.int32Values = {sMockSignal};
265 mTurnSignalValue.prop = 0;
266 }
267 } else {
268 // While testing without a vehicle, behave as if we're in reverse for the first 20 seconds
269 static const int kShowTime = 20; // seconds
270
271 // See if it's time to turn off the default reverse camera
272 static std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
273 std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();
274 if (std::chrono::duration_cast<std::chrono::seconds>(now - start).count() > kShowTime) {
275 // Switch to drive (which should turn off the reverse camera)
276 sMockGear = int32_t(VehicleGear::GEAR_DRIVE);
277 }
278
279 // Build the placeholder vehicle state values (treating single values as 1 element vectors)
280 mGearValue.value.int32Values = {sMockGear};
281 mTurnSignalValue.value.int32Values = {sMockSignal};
282 }
283
284 // Choose our desired EVS state based on the current car state
285 State desiredState = OFF;
286 if (mGearValue.value.int32Values[0] == int32_t(VehicleGear::GEAR_REVERSE)) {
287 desiredState = REVERSE;
288 } else if (mTurnSignalValue.value.int32Values[0] == int32_t(VehicleTurnSignal::RIGHT)) {
289 desiredState = RIGHT;
290 } else if (mTurnSignalValue.value.int32Values[0] == int32_t(VehicleTurnSignal::LEFT)) {
291 desiredState = LEFT;
292 } else if (mGearValue.value.int32Values[0] == int32_t(VehicleGear::GEAR_PARK)) {
293 desiredState = PARKING;
294 }
295
296 // Apply the desire state
297 return configureEvsPipeline(desiredState);
298 }
299
invokeGet(VehiclePropValue * pRequestedPropValue)300 ErrorCode EvsStateControl::invokeGet(VehiclePropValue* pRequestedPropValue) {
301 auto halPropValue = mVehicle->createHalPropValue(pRequestedPropValue->prop);
302 // We are only setting int32Values.
303 halPropValue->setInt32Values(pRequestedPropValue->value.int32Values);
304
305 VhalClientResult<std::unique_ptr<IHalPropValue>> result = mVehicle->getValueSync(*halPropValue);
306
307 if (!result.ok()) {
308 return static_cast<ErrorCode>(result.error().code());
309 }
310 pRequestedPropValue->value.int32Values = result.value()->getInt32Values();
311 pRequestedPropValue->timestamp = result.value()->getTimestamp();
312 return ErrorCode::OK;
313 }
314
configureEvsPipeline(State desiredState)315 bool EvsStateControl::configureEvsPipeline(State desiredState) {
316 static bool isGlReady = false;
317
318 if (mCurrentState == desiredState) {
319 // Nothing to do here...
320 return true;
321 }
322
323 // Used by CarStats to accurately compute stats, it needs to be close to the beginning.
324 auto desiredStateTimeMillis = android::uptimeMillis();
325
326 LOG(DEBUG) << "Switching to state " << desiredState;
327 LOG(DEBUG) << " Current state " << mCurrentState << " has "
328 << mCameraList[mCurrentState].size() << " cameras";
329 LOG(DEBUG) << " Desired state " << desiredState << " has " << mCameraList[desiredState].size()
330 << " cameras";
331
332 if (!isGlReady && !isSfReady()) {
333 // Graphics is not ready yet; using CPU renderer.
334 if (mCameraList[desiredState].size() >= 1) {
335 mDesiredRenderer =
336 std::make_unique<RenderPixelCopy>(mEvs, mCameraList[desiredState][0]);
337 if (!mDesiredRenderer) {
338 LOG(ERROR) << "Failed to construct Pixel Copy renderer. Skipping state change.";
339 return false;
340 }
341 } else {
342 LOG(DEBUG) << "Unsupported, desiredState " << desiredState << " has "
343 << mCameraList[desiredState].size() << " cameras.";
344 }
345 } else {
346 // Assumes that SurfaceFlinger is available always after being launched.
347
348 // Do we need a new direct view renderer?
349 if (mCameraList[desiredState].size() == 1) {
350 // We have a camera assigned to this state for direct view.
351 mDesiredRenderer =
352 std::make_unique<RenderDirectView>(mEvs, mCameraDescList[desiredState][0],
353 mConfig);
354 if (!mDesiredRenderer) {
355 LOG(ERROR) << "Failed to construct direct renderer. Skipping state change.";
356 return false;
357 }
358 } else if (mCameraList[desiredState].size() > 1 ||
359 (mCameraList[desiredState].size() > 0 && desiredState == PARKING)) {
360 // TODO(b/140668179): RenderTopView needs to be updated to use new
361 // ConfigManager.
362 mDesiredRenderer =
363 std::make_unique<RenderTopView>(mEvs, mCameraList[desiredState], mConfig);
364 if (!mDesiredRenderer) {
365 LOG(ERROR) << "Failed to construct top view renderer. Skipping state change.";
366 return false;
367 }
368 } else {
369 LOG(DEBUG) << "Unsupported, desiredState " << desiredState << " has "
370 << mCameraList[desiredState].size() << " cameras.";
371 }
372
373 // GL renderer is now ready.
374 isGlReady = true;
375 }
376
377 // Since we're changing states, shut down the current renderer
378 if (mCurrentRenderer) {
379 mCurrentRenderer->deactivate();
380 mCurrentRenderer.reset();
381 }
382
383 // Now set the display state based on whether we have a video feed to show
384 std::shared_ptr<IEvsDisplay> displayHandle = mDisplay.lock();
385 if (!displayHandle) {
386 return false;
387 }
388
389 if (!mDesiredRenderer) {
390 LOG(DEBUG) << "Turning off the display";
391 displayHandle->setDisplayState(DisplayState::NOT_VISIBLE);
392 } else {
393 mCurrentRenderer = std::move(mDesiredRenderer);
394
395 // Start the camera stream
396 LOG(DEBUG) << "EvsStartCameraStreamTiming start time: " << android::elapsedRealtime()
397 << " ms.";
398 if (!mCurrentRenderer->activate()) {
399 LOG(ERROR) << "New renderer failed to activate";
400 return false;
401 }
402
403 // Activate the display
404 LOG(DEBUG) << "EvsActivateDisplayTiming start time: " << android::elapsedRealtime()
405 << " ms.";
406 if (auto status = displayHandle->setDisplayState(DisplayState::VISIBLE_ON_NEXT_FRAME);
407 !status.isOk()) {
408 LOG(ERROR) << "Failed to set a display state as VISIBLE_ON_NEXT_FRAME.";
409 return false;
410 }
411 }
412
413 // Record our current state
414 LOG(INFO) << "Activated state " << desiredState;
415 mCurrentState = desiredState;
416
417 mFirstFrameIsDisplayed = false; // Got a new renderer, mark first frame is not displayed.
418
419 if (mCurrentRenderer && desiredState == State::REVERSE) {
420 // Start computing the latency when the evs state changes.
421 mEvsStats.startComputingFirstFrameLatency(desiredStateTimeMillis);
422 }
423
424 return true;
425 }
426