1 /**
2 * Copyright 2024 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <cstdint>
18 #define LOG_TAG "InputTransport"
19 #define ATRACE_TAG ATRACE_TAG_INPUT
20
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <inttypes.h>
24 #include <math.h>
25 #include <poll.h>
26 #include <sys/socket.h>
27 #include <sys/types.h>
28 #include <unistd.h>
29
30 #include <android-base/logging.h>
31 #include <android-base/properties.h>
32 #include <android-base/stringprintf.h>
33 #include <binder/Parcel.h>
34 #include <cutils/properties.h>
35 #include <ftl/enum.h>
36 #include <log/log.h>
37 #include <utils/Trace.h>
38
39 #include <com_android_input_flags.h>
40 #include <input/InputConsumer.h>
41 #include <input/PrintTools.h>
42 #include <input/TraceTools.h>
43
44 namespace input_flags = com::android::input::flags;
45
46 namespace android {
47
48 namespace {
49
50 /**
51 * Log debug messages relating to the consumer end of the transport channel.
52 * Enable this via "adb shell setprop log.tag.InputTransportConsumer DEBUG" (requires restart)
53 */
54
55 const bool DEBUG_TRANSPORT_CONSUMER =
56 __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Consumer", ANDROID_LOG_INFO);
57
58 const bool IS_DEBUGGABLE_BUILD =
59 #if defined(__ANDROID__)
60 android::base::GetBoolProperty("ro.debuggable", false);
61 #else
62 true;
63 #endif
64
65 /**
66 * Log debug messages about touch event resampling.
67 *
68 * Enable this via "adb shell setprop log.tag.InputTransportResampling DEBUG".
69 * This requires a restart on non-debuggable (e.g. user) builds, but should take effect immediately
70 * on debuggable builds (e.g. userdebug).
71 */
debugResampling()72 bool debugResampling() {
73 if (!IS_DEBUGGABLE_BUILD) {
74 static const bool DEBUG_TRANSPORT_RESAMPLING =
75 __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling",
76 ANDROID_LOG_INFO);
77 return DEBUG_TRANSPORT_RESAMPLING;
78 }
79 return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO);
80 }
81
initializeKeyEvent(KeyEvent & event,const InputMessage & msg)82 void initializeKeyEvent(KeyEvent& event, const InputMessage& msg) {
83 event.initialize(msg.body.key.eventId, msg.body.key.deviceId, msg.body.key.source,
84 ui::LogicalDisplayId{msg.body.key.displayId}, msg.body.key.hmac,
85 msg.body.key.action, msg.body.key.flags, msg.body.key.keyCode,
86 msg.body.key.scanCode, msg.body.key.metaState, msg.body.key.repeatCount,
87 msg.body.key.downTime, msg.body.key.eventTime);
88 }
89
initializeFocusEvent(FocusEvent & event,const InputMessage & msg)90 void initializeFocusEvent(FocusEvent& event, const InputMessage& msg) {
91 event.initialize(msg.body.focus.eventId, msg.body.focus.hasFocus);
92 }
93
initializeCaptureEvent(CaptureEvent & event,const InputMessage & msg)94 void initializeCaptureEvent(CaptureEvent& event, const InputMessage& msg) {
95 event.initialize(msg.body.capture.eventId, msg.body.capture.pointerCaptureEnabled);
96 }
97
initializeDragEvent(DragEvent & event,const InputMessage & msg)98 void initializeDragEvent(DragEvent& event, const InputMessage& msg) {
99 event.initialize(msg.body.drag.eventId, msg.body.drag.x, msg.body.drag.y,
100 msg.body.drag.isExiting);
101 }
102
initializeMotionEvent(MotionEvent & event,const InputMessage & msg)103 void initializeMotionEvent(MotionEvent& event, const InputMessage& msg) {
104 uint32_t pointerCount = msg.body.motion.pointerCount;
105 PointerProperties pointerProperties[pointerCount];
106 PointerCoords pointerCoords[pointerCount];
107 for (uint32_t i = 0; i < pointerCount; i++) {
108 pointerProperties[i] = msg.body.motion.pointers[i].properties;
109 pointerCoords[i] = msg.body.motion.pointers[i].coords;
110 }
111
112 ui::Transform transform;
113 transform.set({msg.body.motion.dsdx, msg.body.motion.dtdx, msg.body.motion.tx,
114 msg.body.motion.dtdy, msg.body.motion.dsdy, msg.body.motion.ty, 0, 0, 1});
115 ui::Transform displayTransform;
116 displayTransform.set({msg.body.motion.dsdxRaw, msg.body.motion.dtdxRaw, msg.body.motion.txRaw,
117 msg.body.motion.dtdyRaw, msg.body.motion.dsdyRaw, msg.body.motion.tyRaw,
118 0, 0, 1});
119 event.initialize(msg.body.motion.eventId, msg.body.motion.deviceId, msg.body.motion.source,
120 ui::LogicalDisplayId{msg.body.motion.displayId}, msg.body.motion.hmac,
121 msg.body.motion.action, msg.body.motion.actionButton, msg.body.motion.flags,
122 msg.body.motion.edgeFlags, msg.body.motion.metaState,
123 msg.body.motion.buttonState, msg.body.motion.classification, transform,
124 msg.body.motion.xPrecision, msg.body.motion.yPrecision,
125 msg.body.motion.xCursorPosition, msg.body.motion.yCursorPosition,
126 displayTransform, msg.body.motion.downTime, msg.body.motion.eventTime,
127 pointerCount, pointerProperties, pointerCoords);
128 }
129
addSample(MotionEvent & event,const InputMessage & msg)130 void addSample(MotionEvent& event, const InputMessage& msg) {
131 uint32_t pointerCount = msg.body.motion.pointerCount;
132 PointerCoords pointerCoords[pointerCount];
133 for (uint32_t i = 0; i < pointerCount; i++) {
134 pointerCoords[i] = msg.body.motion.pointers[i].coords;
135 }
136
137 event.setMetaState(event.getMetaState() | msg.body.motion.metaState);
138 event.addSample(msg.body.motion.eventTime, pointerCoords);
139 }
140
initializeTouchModeEvent(TouchModeEvent & event,const InputMessage & msg)141 void initializeTouchModeEvent(TouchModeEvent& event, const InputMessage& msg) {
142 event.initialize(msg.body.touchMode.eventId, msg.body.touchMode.isInTouchMode);
143 }
144
145 // Nanoseconds per milliseconds.
146 constexpr nsecs_t NANOS_PER_MS = 1000000;
147
148 // Latency added during resampling. A few milliseconds doesn't hurt much but
149 // reduces the impact of mispredicted touch positions.
150 const std::chrono::duration RESAMPLE_LATENCY = 5ms;
151
152 // Minimum time difference between consecutive samples before attempting to resample.
153 const nsecs_t RESAMPLE_MIN_DELTA = 2 * NANOS_PER_MS;
154
155 // Maximum time difference between consecutive samples before attempting to resample
156 // by extrapolation.
157 const nsecs_t RESAMPLE_MAX_DELTA = 20 * NANOS_PER_MS;
158
159 // Maximum time to predict forward from the last known state, to avoid predicting too
160 // far into the future. This time is further bounded by 50% of the last time delta.
161 const nsecs_t RESAMPLE_MAX_PREDICTION = 8 * NANOS_PER_MS;
162
163 /**
164 * System property for enabling / disabling touch resampling.
165 * Resampling extrapolates / interpolates the reported touch event coordinates to better
166 * align them to the VSYNC signal, thus resulting in smoother scrolling performance.
167 * Resampling is not needed (and should be disabled) on hardware that already
168 * has touch events triggered by VSYNC.
169 * Set to "1" to enable resampling (default).
170 * Set to "0" to disable resampling.
171 * Resampling is enabled by default.
172 */
173 const char* PROPERTY_RESAMPLING_ENABLED = "ro.input.resampling";
174
lerp(float a,float b,float alpha)175 inline float lerp(float a, float b, float alpha) {
176 return a + alpha * (b - a);
177 }
178
isPointerEvent(int32_t source)179 inline bool isPointerEvent(int32_t source) {
180 return (source & AINPUT_SOURCE_CLASS_POINTER) == AINPUT_SOURCE_CLASS_POINTER;
181 }
182
shouldResampleTool(ToolType toolType)183 bool shouldResampleTool(ToolType toolType) {
184 return toolType == ToolType::FINGER || toolType == ToolType::MOUSE ||
185 toolType == ToolType::STYLUS || toolType == ToolType::UNKNOWN;
186 }
187
188 } // namespace
189
190 using android::base::Result;
191 using android::base::StringPrintf;
192
193 // --- InputConsumer ---
194
InputConsumer(const std::shared_ptr<InputChannel> & channel)195 InputConsumer::InputConsumer(const std::shared_ptr<InputChannel>& channel)
196 : InputConsumer(channel, isTouchResamplingEnabled()) {}
197
InputConsumer(const std::shared_ptr<InputChannel> & channel,bool enableTouchResampling)198 InputConsumer::InputConsumer(const std::shared_ptr<InputChannel>& channel,
199 bool enableTouchResampling)
200 : mResampleTouch(enableTouchResampling),
201 mChannel(channel),
202 mProcessingTraceTag(StringPrintf("InputConsumer processing on %s (%p)",
203 mChannel->getName().c_str(), this)),
204 mLifetimeTraceTag(StringPrintf("InputConsumer lifetime on %s (%p)",
205 mChannel->getName().c_str(), this)),
206 mLifetimeTraceCookie(
207 static_cast<int32_t>(reinterpret_cast<std::uintptr_t>(this) & 0xFFFFFFFF)),
208 mMsgDeferred(false) {
209 ATRACE_ASYNC_BEGIN(mLifetimeTraceTag.c_str(), /*cookie=*/mLifetimeTraceCookie);
210 }
211
~InputConsumer()212 InputConsumer::~InputConsumer() {
213 ATRACE_ASYNC_END(mLifetimeTraceTag.c_str(), /*cookie=*/mLifetimeTraceCookie);
214 }
215
isTouchResamplingEnabled()216 bool InputConsumer::isTouchResamplingEnabled() {
217 return property_get_bool(PROPERTY_RESAMPLING_ENABLED, true);
218 }
219
consume(InputEventFactoryInterface * factory,bool consumeBatches,nsecs_t frameTime,uint32_t * outSeq,InputEvent ** outEvent)220 status_t InputConsumer::consume(InputEventFactoryInterface* factory, bool consumeBatches,
221 nsecs_t frameTime, uint32_t* outSeq, InputEvent** outEvent) {
222 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
223 "channel '%s' consumer ~ consume: consumeBatches=%s, frameTime=%" PRId64,
224 mChannel->getName().c_str(), toString(consumeBatches), frameTime);
225
226 *outSeq = 0;
227 *outEvent = nullptr;
228
229 // Fetch the next input message.
230 // Loop until an event can be returned or no additional events are received.
231 while (!*outEvent) {
232 if (mMsgDeferred) {
233 // mMsg contains a valid input message from the previous call to consume
234 // that has not yet been processed.
235 mMsgDeferred = false;
236 } else {
237 // Receive a fresh message.
238 status_t result = mChannel->receiveMessage(&mMsg);
239 if (result == OK) {
240 const auto [_, inserted] =
241 mConsumeTimes.emplace(mMsg.header.seq, systemTime(SYSTEM_TIME_MONOTONIC));
242 LOG_ALWAYS_FATAL_IF(!inserted, "Already have a consume time for seq=%" PRIu32,
243 mMsg.header.seq);
244
245 // Trace the event processing timeline - event was just read from the socket
246 ATRACE_ASYNC_BEGIN(mProcessingTraceTag.c_str(), /*cookie=*/mMsg.header.seq);
247 }
248 if (result) {
249 // Consume the next batched event unless batches are being held for later.
250 if (consumeBatches || result != WOULD_BLOCK) {
251 result = consumeBatch(factory, frameTime, outSeq, outEvent);
252 if (*outEvent) {
253 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
254 "channel '%s' consumer ~ consumed batch event, seq=%u",
255 mChannel->getName().c_str(), *outSeq);
256 break;
257 }
258 }
259 return result;
260 }
261 }
262
263 switch (mMsg.header.type) {
264 case InputMessage::Type::KEY: {
265 KeyEvent* keyEvent = factory->createKeyEvent();
266 if (!keyEvent) return NO_MEMORY;
267
268 initializeKeyEvent(*keyEvent, mMsg);
269 *outSeq = mMsg.header.seq;
270 *outEvent = keyEvent;
271 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
272 "channel '%s' consumer ~ consumed key event, seq=%u",
273 mChannel->getName().c_str(), *outSeq);
274 break;
275 }
276
277 case InputMessage::Type::MOTION: {
278 ssize_t batchIndex = findBatch(mMsg.body.motion.deviceId, mMsg.body.motion.source);
279 if (batchIndex >= 0) {
280 Batch& batch = mBatches[batchIndex];
281 if (canAddSample(batch, &mMsg)) {
282 batch.samples.push_back(mMsg);
283 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
284 "channel '%s' consumer ~ appended to batch event",
285 mChannel->getName().c_str());
286 break;
287 } else if (isPointerEvent(mMsg.body.motion.source) &&
288 mMsg.body.motion.action == AMOTION_EVENT_ACTION_CANCEL) {
289 // No need to process events that we are going to cancel anyways
290 const size_t count = batch.samples.size();
291 for (size_t i = 0; i < count; i++) {
292 const InputMessage& msg = batch.samples[i];
293 sendFinishedSignal(msg.header.seq, false);
294 }
295 batch.samples.erase(batch.samples.begin(), batch.samples.begin() + count);
296 mBatches.erase(mBatches.begin() + batchIndex);
297 } else {
298 // We cannot append to the batch in progress, so we need to consume
299 // the previous batch right now and defer the new message until later.
300 mMsgDeferred = true;
301 status_t result = consumeSamples(factory, batch, batch.samples.size(),
302 outSeq, outEvent);
303 mBatches.erase(mBatches.begin() + batchIndex);
304 if (result) {
305 return result;
306 }
307 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
308 "channel '%s' consumer ~ consumed batch event and "
309 "deferred current event, seq=%u",
310 mChannel->getName().c_str(), *outSeq);
311 break;
312 }
313 }
314
315 // Start a new batch if needed.
316 if (mMsg.body.motion.action == AMOTION_EVENT_ACTION_MOVE ||
317 mMsg.body.motion.action == AMOTION_EVENT_ACTION_HOVER_MOVE) {
318 Batch batch;
319 batch.samples.push_back(mMsg);
320 mBatches.push_back(batch);
321 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
322 "channel '%s' consumer ~ started batch event",
323 mChannel->getName().c_str());
324 break;
325 }
326
327 MotionEvent* motionEvent = factory->createMotionEvent();
328 if (!motionEvent) return NO_MEMORY;
329
330 updateTouchState(mMsg);
331 initializeMotionEvent(*motionEvent, mMsg);
332 *outSeq = mMsg.header.seq;
333 *outEvent = motionEvent;
334
335 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
336 "channel '%s' consumer ~ consumed motion event, seq=%u",
337 mChannel->getName().c_str(), *outSeq);
338 break;
339 }
340
341 case InputMessage::Type::FINISHED:
342 case InputMessage::Type::TIMELINE: {
343 LOG(FATAL) << "Consumed a " << ftl::enum_string(mMsg.header.type)
344 << " message, which should never be seen by "
345 "InputConsumer on "
346 << mChannel->getName();
347 break;
348 }
349
350 case InputMessage::Type::FOCUS: {
351 FocusEvent* focusEvent = factory->createFocusEvent();
352 if (!focusEvent) return NO_MEMORY;
353
354 initializeFocusEvent(*focusEvent, mMsg);
355 *outSeq = mMsg.header.seq;
356 *outEvent = focusEvent;
357 break;
358 }
359
360 case InputMessage::Type::CAPTURE: {
361 CaptureEvent* captureEvent = factory->createCaptureEvent();
362 if (!captureEvent) return NO_MEMORY;
363
364 initializeCaptureEvent(*captureEvent, mMsg);
365 *outSeq = mMsg.header.seq;
366 *outEvent = captureEvent;
367 break;
368 }
369
370 case InputMessage::Type::DRAG: {
371 DragEvent* dragEvent = factory->createDragEvent();
372 if (!dragEvent) return NO_MEMORY;
373
374 initializeDragEvent(*dragEvent, mMsg);
375 *outSeq = mMsg.header.seq;
376 *outEvent = dragEvent;
377 break;
378 }
379
380 case InputMessage::Type::TOUCH_MODE: {
381 TouchModeEvent* touchModeEvent = factory->createTouchModeEvent();
382 if (!touchModeEvent) return NO_MEMORY;
383
384 initializeTouchModeEvent(*touchModeEvent, mMsg);
385 *outSeq = mMsg.header.seq;
386 *outEvent = touchModeEvent;
387 break;
388 }
389 }
390 }
391 return OK;
392 }
393
consumeBatch(InputEventFactoryInterface * factory,nsecs_t frameTime,uint32_t * outSeq,InputEvent ** outEvent)394 status_t InputConsumer::consumeBatch(InputEventFactoryInterface* factory, nsecs_t frameTime,
395 uint32_t* outSeq, InputEvent** outEvent) {
396 status_t result;
397 for (size_t i = mBatches.size(); i > 0;) {
398 i--;
399 Batch& batch = mBatches[i];
400 if (frameTime < 0) {
401 result = consumeSamples(factory, batch, batch.samples.size(), outSeq, outEvent);
402 mBatches.erase(mBatches.begin() + i);
403 return result;
404 }
405
406 nsecs_t sampleTime = frameTime;
407 if (mResampleTouch) {
408 sampleTime -= std::chrono::nanoseconds(RESAMPLE_LATENCY).count();
409 }
410 ssize_t split = findSampleNoLaterThan(batch, sampleTime);
411 if (split < 0) {
412 continue;
413 }
414
415 result = consumeSamples(factory, batch, split + 1, outSeq, outEvent);
416 const InputMessage* next;
417 if (batch.samples.empty()) {
418 mBatches.erase(mBatches.begin() + i);
419 next = nullptr;
420 } else {
421 next = &batch.samples[0];
422 }
423 if (!result && mResampleTouch) {
424 resampleTouchState(sampleTime, static_cast<MotionEvent*>(*outEvent), next);
425 }
426 return result;
427 }
428
429 return WOULD_BLOCK;
430 }
431
consumeSamples(InputEventFactoryInterface * factory,Batch & batch,size_t count,uint32_t * outSeq,InputEvent ** outEvent)432 status_t InputConsumer::consumeSamples(InputEventFactoryInterface* factory, Batch& batch,
433 size_t count, uint32_t* outSeq, InputEvent** outEvent) {
434 MotionEvent* motionEvent = factory->createMotionEvent();
435 if (!motionEvent) return NO_MEMORY;
436
437 uint32_t chain = 0;
438 for (size_t i = 0; i < count; i++) {
439 InputMessage& msg = batch.samples[i];
440 updateTouchState(msg);
441 if (i) {
442 SeqChain seqChain;
443 seqChain.seq = msg.header.seq;
444 seqChain.chain = chain;
445 mSeqChains.push_back(seqChain);
446 addSample(*motionEvent, msg);
447 } else {
448 initializeMotionEvent(*motionEvent, msg);
449 }
450 chain = msg.header.seq;
451 }
452 batch.samples.erase(batch.samples.begin(), batch.samples.begin() + count);
453
454 *outSeq = chain;
455 *outEvent = motionEvent;
456 return OK;
457 }
458
updateTouchState(InputMessage & msg)459 void InputConsumer::updateTouchState(InputMessage& msg) {
460 if (!mResampleTouch || !isPointerEvent(msg.body.motion.source)) {
461 return;
462 }
463
464 int32_t deviceId = msg.body.motion.deviceId;
465 int32_t source = msg.body.motion.source;
466
467 // Update the touch state history to incorporate the new input message.
468 // If the message is in the past relative to the most recently produced resampled
469 // touch, then use the resampled time and coordinates instead.
470 switch (msg.body.motion.action & AMOTION_EVENT_ACTION_MASK) {
471 case AMOTION_EVENT_ACTION_DOWN: {
472 ssize_t index = findTouchState(deviceId, source);
473 if (index < 0) {
474 mTouchStates.push_back({});
475 index = mTouchStates.size() - 1;
476 }
477 TouchState& touchState = mTouchStates[index];
478 touchState.initialize(deviceId, source);
479 touchState.addHistory(msg);
480 break;
481 }
482
483 case AMOTION_EVENT_ACTION_MOVE: {
484 ssize_t index = findTouchState(deviceId, source);
485 if (index >= 0) {
486 TouchState& touchState = mTouchStates[index];
487 touchState.addHistory(msg);
488 rewriteMessage(touchState, msg);
489 }
490 break;
491 }
492
493 case AMOTION_EVENT_ACTION_POINTER_DOWN: {
494 ssize_t index = findTouchState(deviceId, source);
495 if (index >= 0) {
496 TouchState& touchState = mTouchStates[index];
497 touchState.lastResample.idBits.clearBit(msg.body.motion.getActionId());
498 rewriteMessage(touchState, msg);
499 }
500 break;
501 }
502
503 case AMOTION_EVENT_ACTION_POINTER_UP: {
504 ssize_t index = findTouchState(deviceId, source);
505 if (index >= 0) {
506 TouchState& touchState = mTouchStates[index];
507 rewriteMessage(touchState, msg);
508 touchState.lastResample.idBits.clearBit(msg.body.motion.getActionId());
509 }
510 break;
511 }
512
513 case AMOTION_EVENT_ACTION_SCROLL: {
514 ssize_t index = findTouchState(deviceId, source);
515 if (index >= 0) {
516 TouchState& touchState = mTouchStates[index];
517 rewriteMessage(touchState, msg);
518 }
519 break;
520 }
521
522 case AMOTION_EVENT_ACTION_UP:
523 case AMOTION_EVENT_ACTION_CANCEL: {
524 ssize_t index = findTouchState(deviceId, source);
525 if (index >= 0) {
526 TouchState& touchState = mTouchStates[index];
527 rewriteMessage(touchState, msg);
528 mTouchStates.erase(mTouchStates.begin() + index);
529 }
530 break;
531 }
532 }
533 }
534
535 /**
536 * Replace the coordinates in msg with the coordinates in lastResample, if necessary.
537 *
538 * If lastResample is no longer valid for a specific pointer (i.e. the lastResample time
539 * is in the past relative to msg and the past two events do not contain identical coordinates),
540 * then invalidate the lastResample data for that pointer.
541 * If the two past events have identical coordinates, then lastResample data for that pointer will
542 * remain valid, and will be used to replace these coordinates. Thus, if a certain coordinate x0 is
543 * resampled to the new value x1, then x1 will always be used to replace x0 until some new value
544 * not equal to x0 is received.
545 */
rewriteMessage(TouchState & state,InputMessage & msg)546 void InputConsumer::rewriteMessage(TouchState& state, InputMessage& msg) {
547 nsecs_t eventTime = msg.body.motion.eventTime;
548 for (uint32_t i = 0; i < msg.body.motion.pointerCount; i++) {
549 uint32_t id = msg.body.motion.pointers[i].properties.id;
550 if (state.lastResample.idBits.hasBit(id)) {
551 if (eventTime < state.lastResample.eventTime ||
552 state.recentCoordinatesAreIdentical(id)) {
553 PointerCoords& msgCoords = msg.body.motion.pointers[i].coords;
554 const PointerCoords& resampleCoords = state.lastResample.getPointerById(id);
555 ALOGD_IF(debugResampling(), "[%d] - rewrite (%0.3f, %0.3f), old (%0.3f, %0.3f)", id,
556 resampleCoords.getX(), resampleCoords.getY(), msgCoords.getX(),
557 msgCoords.getY());
558 msgCoords.setAxisValue(AMOTION_EVENT_AXIS_X, resampleCoords.getX());
559 msgCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, resampleCoords.getY());
560 msgCoords.isResampled = true;
561 } else {
562 state.lastResample.idBits.clearBit(id);
563 }
564 }
565 }
566 }
567
resampleTouchState(nsecs_t sampleTime,MotionEvent * event,const InputMessage * next)568 void InputConsumer::resampleTouchState(nsecs_t sampleTime, MotionEvent* event,
569 const InputMessage* next) {
570 if (!mResampleTouch || !(isPointerEvent(event->getSource())) ||
571 event->getAction() != AMOTION_EVENT_ACTION_MOVE) {
572 return;
573 }
574
575 ssize_t index = findTouchState(event->getDeviceId(), event->getSource());
576 if (index < 0) {
577 ALOGD_IF(debugResampling(), "Not resampled, no touch state for device.");
578 return;
579 }
580
581 TouchState& touchState = mTouchStates[index];
582 if (touchState.historySize < 1) {
583 ALOGD_IF(debugResampling(), "Not resampled, no history for device.");
584 return;
585 }
586
587 // Ensure that the current sample has all of the pointers that need to be reported.
588 const History* current = touchState.getHistory(0);
589 size_t pointerCount = event->getPointerCount();
590 for (size_t i = 0; i < pointerCount; i++) {
591 uint32_t id = event->getPointerId(i);
592 if (!current->idBits.hasBit(id)) {
593 ALOGD_IF(debugResampling(), "Not resampled, missing id %d", id);
594 return;
595 }
596 if (!shouldResampleTool(event->getToolType(i))) {
597 ALOGD_IF(debugResampling(),
598 "Not resampled, containing unsupported tool type at pointer %d", id);
599 return;
600 }
601 }
602
603 // Find the data to use for resampling.
604 const History* other;
605 History future;
606 float alpha;
607 if (next) {
608 // Interpolate between current sample and future sample.
609 // So current->eventTime <= sampleTime <= future.eventTime.
610 future.initializeFrom(*next);
611 other = &future;
612 nsecs_t delta = future.eventTime - current->eventTime;
613 if (delta < RESAMPLE_MIN_DELTA) {
614 ALOGD_IF(debugResampling(), "Not resampled, delta time is too small: %" PRId64 " ns.",
615 delta);
616 return;
617 }
618 alpha = float(sampleTime - current->eventTime) / delta;
619 } else if (touchState.historySize >= 2) {
620 // Extrapolate future sample using current sample and past sample.
621 // So other->eventTime <= current->eventTime <= sampleTime.
622 other = touchState.getHistory(1);
623 nsecs_t delta = current->eventTime - other->eventTime;
624 if (delta < RESAMPLE_MIN_DELTA) {
625 ALOGD_IF(debugResampling(), "Not resampled, delta time is too small: %" PRId64 " ns.",
626 delta);
627 return;
628 } else if (delta > RESAMPLE_MAX_DELTA) {
629 ALOGD_IF(debugResampling(), "Not resampled, delta time is too large: %" PRId64 " ns.",
630 delta);
631 return;
632 }
633 nsecs_t maxPredict = current->eventTime + std::min(delta / 2, RESAMPLE_MAX_PREDICTION);
634 if (sampleTime > maxPredict) {
635 ALOGD_IF(debugResampling(),
636 "Sample time is too far in the future, adjusting prediction "
637 "from %" PRId64 " to %" PRId64 " ns.",
638 sampleTime - current->eventTime, maxPredict - current->eventTime);
639 sampleTime = maxPredict;
640 }
641 alpha = float(current->eventTime - sampleTime) / delta;
642 } else {
643 ALOGD_IF(debugResampling(), "Not resampled, insufficient data.");
644 return;
645 }
646
647 if (current->eventTime == sampleTime) {
648 ALOGD_IF(debugResampling(), "Not resampled, 2 events with identical times.");
649 return;
650 }
651
652 for (size_t i = 0; i < pointerCount; i++) {
653 uint32_t id = event->getPointerId(i);
654 if (!other->idBits.hasBit(id)) {
655 ALOGD_IF(debugResampling(), "Not resampled, the other doesn't have pointer id %d.", id);
656 return;
657 }
658 }
659
660 // Resample touch coordinates.
661 History oldLastResample;
662 oldLastResample.initializeFrom(touchState.lastResample);
663 touchState.lastResample.eventTime = sampleTime;
664 touchState.lastResample.idBits.clear();
665 for (size_t i = 0; i < pointerCount; i++) {
666 uint32_t id = event->getPointerId(i);
667 touchState.lastResample.idToIndex[id] = i;
668 touchState.lastResample.idBits.markBit(id);
669 if (oldLastResample.hasPointerId(id) && touchState.recentCoordinatesAreIdentical(id)) {
670 // We maintain the previously resampled value for this pointer (stored in
671 // oldLastResample) when the coordinates for this pointer haven't changed since then.
672 // This way we don't introduce artificial jitter when pointers haven't actually moved.
673 // The isResampled flag isn't cleared as the values don't reflect what the device is
674 // actually reporting.
675
676 // We know here that the coordinates for the pointer haven't changed because we
677 // would've cleared the resampled bit in rewriteMessage if they had. We can't modify
678 // lastResample in place because the mapping from pointer ID to index may have changed.
679 touchState.lastResample.pointers[i] = oldLastResample.getPointerById(id);
680 continue;
681 }
682
683 PointerCoords& resampledCoords = touchState.lastResample.pointers[i];
684 const PointerCoords& currentCoords = current->getPointerById(id);
685 resampledCoords = currentCoords;
686 resampledCoords.isResampled = true;
687 const PointerCoords& otherCoords = other->getPointerById(id);
688 resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_X,
689 lerp(currentCoords.getX(), otherCoords.getX(), alpha));
690 resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_Y,
691 lerp(currentCoords.getY(), otherCoords.getY(), alpha));
692 ALOGD_IF(debugResampling(),
693 "[%d] - out (%0.3f, %0.3f), cur (%0.3f, %0.3f), "
694 "other (%0.3f, %0.3f), alpha %0.3f",
695 id, resampledCoords.getX(), resampledCoords.getY(), currentCoords.getX(),
696 currentCoords.getY(), otherCoords.getX(), otherCoords.getY(), alpha);
697 }
698
699 event->addSample(sampleTime, touchState.lastResample.pointers);
700 }
701
sendFinishedSignal(uint32_t seq,bool handled)702 status_t InputConsumer::sendFinishedSignal(uint32_t seq, bool handled) {
703 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
704 "channel '%s' consumer ~ sendFinishedSignal: seq=%u, handled=%s",
705 mChannel->getName().c_str(), seq, toString(handled));
706
707 if (!seq) {
708 ALOGE("Attempted to send a finished signal with sequence number 0.");
709 return BAD_VALUE;
710 }
711
712 // Send finished signals for the batch sequence chain first.
713 size_t seqChainCount = mSeqChains.size();
714 if (seqChainCount) {
715 uint32_t currentSeq = seq;
716 uint32_t chainSeqs[seqChainCount];
717 size_t chainIndex = 0;
718 for (size_t i = seqChainCount; i > 0;) {
719 i--;
720 const SeqChain& seqChain = mSeqChains[i];
721 if (seqChain.seq == currentSeq) {
722 currentSeq = seqChain.chain;
723 chainSeqs[chainIndex++] = currentSeq;
724 mSeqChains.erase(mSeqChains.begin() + i);
725 }
726 }
727 status_t status = OK;
728 while (!status && chainIndex > 0) {
729 chainIndex--;
730 status = sendUnchainedFinishedSignal(chainSeqs[chainIndex], handled);
731 }
732 if (status) {
733 // An error occurred so at least one signal was not sent, reconstruct the chain.
734 for (;;) {
735 SeqChain seqChain;
736 seqChain.seq = chainIndex != 0 ? chainSeqs[chainIndex - 1] : seq;
737 seqChain.chain = chainSeqs[chainIndex];
738 mSeqChains.push_back(seqChain);
739 if (!chainIndex) break;
740 chainIndex--;
741 }
742 return status;
743 }
744 }
745
746 // Send finished signal for the last message in the batch.
747 return sendUnchainedFinishedSignal(seq, handled);
748 }
749
sendTimeline(int32_t inputEventId,std::array<nsecs_t,GraphicsTimeline::SIZE> graphicsTimeline)750 status_t InputConsumer::sendTimeline(int32_t inputEventId,
751 std::array<nsecs_t, GraphicsTimeline::SIZE> graphicsTimeline) {
752 ALOGD_IF(DEBUG_TRANSPORT_CONSUMER,
753 "channel '%s' consumer ~ sendTimeline: inputEventId=%" PRId32
754 ", gpuCompletedTime=%" PRId64 ", presentTime=%" PRId64,
755 mChannel->getName().c_str(), inputEventId,
756 graphicsTimeline[GraphicsTimeline::GPU_COMPLETED_TIME],
757 graphicsTimeline[GraphicsTimeline::PRESENT_TIME]);
758
759 InputMessage msg;
760 msg.header.type = InputMessage::Type::TIMELINE;
761 msg.header.seq = 0;
762 msg.body.timeline.eventId = inputEventId;
763 msg.body.timeline.graphicsTimeline = std::move(graphicsTimeline);
764 return mChannel->sendMessage(&msg);
765 }
766
getConsumeTime(uint32_t seq) const767 nsecs_t InputConsumer::getConsumeTime(uint32_t seq) const {
768 auto it = mConsumeTimes.find(seq);
769 // Consume time will be missing if either 'finishInputEvent' is called twice, or if it was
770 // called for the wrong (synthetic?) input event. Either way, it is a bug that should be fixed.
771 LOG_ALWAYS_FATAL_IF(it == mConsumeTimes.end(), "Could not find consume time for seq=%" PRIu32,
772 seq);
773 return it->second;
774 }
775
popConsumeTime(uint32_t seq)776 void InputConsumer::popConsumeTime(uint32_t seq) {
777 mConsumeTimes.erase(seq);
778 }
779
sendUnchainedFinishedSignal(uint32_t seq,bool handled)780 status_t InputConsumer::sendUnchainedFinishedSignal(uint32_t seq, bool handled) {
781 InputMessage msg;
782 msg.header.type = InputMessage::Type::FINISHED;
783 msg.header.seq = seq;
784 msg.body.finished.handled = handled;
785 msg.body.finished.consumeTime = getConsumeTime(seq);
786 status_t result = mChannel->sendMessage(&msg);
787 if (result == OK) {
788 // Remove the consume time if the socket write succeeded. We will not need to ack this
789 // message anymore. If the socket write did not succeed, we will try again and will still
790 // need consume time.
791 popConsumeTime(seq);
792
793 // Trace the event processing timeline - event was just finished
794 ATRACE_ASYNC_END(mProcessingTraceTag.c_str(), /*cookie=*/seq);
795 }
796 return result;
797 }
798
hasPendingBatch() const799 bool InputConsumer::hasPendingBatch() const {
800 return !mBatches.empty();
801 }
802
getPendingBatchSource() const803 int32_t InputConsumer::getPendingBatchSource() const {
804 if (mBatches.empty()) {
805 return AINPUT_SOURCE_CLASS_NONE;
806 }
807
808 const Batch& batch = mBatches[0];
809 const InputMessage& head = batch.samples[0];
810 return head.body.motion.source;
811 }
812
probablyHasInput() const813 bool InputConsumer::probablyHasInput() const {
814 return hasPendingBatch() || mChannel->probablyHasInput();
815 }
816
findBatch(int32_t deviceId,int32_t source) const817 ssize_t InputConsumer::findBatch(int32_t deviceId, int32_t source) const {
818 for (size_t i = 0; i < mBatches.size(); i++) {
819 const Batch& batch = mBatches[i];
820 const InputMessage& head = batch.samples[0];
821 if (head.body.motion.deviceId == deviceId && head.body.motion.source == source) {
822 return i;
823 }
824 }
825 return -1;
826 }
827
findTouchState(int32_t deviceId,int32_t source) const828 ssize_t InputConsumer::findTouchState(int32_t deviceId, int32_t source) const {
829 for (size_t i = 0; i < mTouchStates.size(); i++) {
830 const TouchState& touchState = mTouchStates[i];
831 if (touchState.deviceId == deviceId && touchState.source == source) {
832 return i;
833 }
834 }
835 return -1;
836 }
837
canAddSample(const Batch & batch,const InputMessage * msg)838 bool InputConsumer::canAddSample(const Batch& batch, const InputMessage* msg) {
839 const InputMessage& head = batch.samples[0];
840 uint32_t pointerCount = msg->body.motion.pointerCount;
841 if (head.body.motion.pointerCount != pointerCount ||
842 head.body.motion.action != msg->body.motion.action) {
843 return false;
844 }
845 for (size_t i = 0; i < pointerCount; i++) {
846 if (head.body.motion.pointers[i].properties != msg->body.motion.pointers[i].properties) {
847 return false;
848 }
849 }
850 return true;
851 }
852
findSampleNoLaterThan(const Batch & batch,nsecs_t time)853 ssize_t InputConsumer::findSampleNoLaterThan(const Batch& batch, nsecs_t time) {
854 size_t numSamples = batch.samples.size();
855 size_t index = 0;
856 while (index < numSamples && batch.samples[index].body.motion.eventTime <= time) {
857 index += 1;
858 }
859 return ssize_t(index) - 1;
860 }
861
dump() const862 std::string InputConsumer::dump() const {
863 std::string out;
864 out = out + "mResampleTouch = " + toString(mResampleTouch) + "\n";
865 out = out + "mChannel = " + mChannel->getName() + "\n";
866 out = out + "mMsgDeferred: " + toString(mMsgDeferred) + "\n";
867 if (mMsgDeferred) {
868 out = out + "mMsg : " + ftl::enum_string(mMsg.header.type) + "\n";
869 }
870 out += "Batches:\n";
871 for (const Batch& batch : mBatches) {
872 out += " Batch:\n";
873 for (const InputMessage& msg : batch.samples) {
874 out += android::base::StringPrintf(" Message %" PRIu32 ": %s ", msg.header.seq,
875 ftl::enum_string(msg.header.type).c_str());
876 switch (msg.header.type) {
877 case InputMessage::Type::KEY: {
878 out += android::base::StringPrintf("action=%s keycode=%" PRId32,
879 KeyEvent::actionToString(
880 msg.body.key.action),
881 msg.body.key.keyCode);
882 break;
883 }
884 case InputMessage::Type::MOTION: {
885 out = out + "action=" + MotionEvent::actionToString(msg.body.motion.action);
886 for (uint32_t i = 0; i < msg.body.motion.pointerCount; i++) {
887 const float x = msg.body.motion.pointers[i].coords.getX();
888 const float y = msg.body.motion.pointers[i].coords.getY();
889 out += android::base::StringPrintf("\n Pointer %" PRIu32
890 " : x=%.1f y=%.1f",
891 i, x, y);
892 }
893 break;
894 }
895 case InputMessage::Type::FINISHED: {
896 out += android::base::StringPrintf("handled=%s, consumeTime=%" PRId64,
897 toString(msg.body.finished.handled),
898 msg.body.finished.consumeTime);
899 break;
900 }
901 case InputMessage::Type::FOCUS: {
902 out += android::base::StringPrintf("hasFocus=%s",
903 toString(msg.body.focus.hasFocus));
904 break;
905 }
906 case InputMessage::Type::CAPTURE: {
907 out += android::base::StringPrintf("hasCapture=%s",
908 toString(msg.body.capture
909 .pointerCaptureEnabled));
910 break;
911 }
912 case InputMessage::Type::DRAG: {
913 out += android::base::StringPrintf("x=%.1f y=%.1f, isExiting=%s",
914 msg.body.drag.x, msg.body.drag.y,
915 toString(msg.body.drag.isExiting));
916 break;
917 }
918 case InputMessage::Type::TIMELINE: {
919 const nsecs_t gpuCompletedTime =
920 msg.body.timeline
921 .graphicsTimeline[GraphicsTimeline::GPU_COMPLETED_TIME];
922 const nsecs_t presentTime =
923 msg.body.timeline.graphicsTimeline[GraphicsTimeline::PRESENT_TIME];
924 out += android::base::StringPrintf("inputEventId=%" PRId32
925 ", gpuCompletedTime=%" PRId64
926 ", presentTime=%" PRId64,
927 msg.body.timeline.eventId, gpuCompletedTime,
928 presentTime);
929 break;
930 }
931 case InputMessage::Type::TOUCH_MODE: {
932 out += android::base::StringPrintf("isInTouchMode=%s",
933 toString(msg.body.touchMode.isInTouchMode));
934 break;
935 }
936 }
937 out += "\n";
938 }
939 }
940 if (mBatches.empty()) {
941 out += " <empty>\n";
942 }
943 out += "mSeqChains:\n";
944 for (const SeqChain& chain : mSeqChains) {
945 out += android::base::StringPrintf(" chain: seq = %" PRIu32 " chain=%" PRIu32, chain.seq,
946 chain.chain);
947 }
948 if (mSeqChains.empty()) {
949 out += " <empty>\n";
950 }
951 out += "mConsumeTimes:\n";
952 for (const auto& [seq, consumeTime] : mConsumeTimes) {
953 out += android::base::StringPrintf(" seq = %" PRIu32 " consumeTime = %" PRId64, seq,
954 consumeTime);
955 }
956 if (mConsumeTimes.empty()) {
957 out += " <empty>\n";
958 }
959 return out;
960 }
961
962 } // namespace android
963