1 /*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "VelocityTracker"
18
19 #include <android-base/logging.h>
20 #include <ftl/enum.h>
21 #include <inttypes.h>
22 #include <limits.h>
23 #include <math.h>
24 #include <array>
25 #include <optional>
26
27 #include <input/PrintTools.h>
28 #include <input/VelocityTracker.h>
29 #include <utils/BitSet.h>
30 #include <utils/Timers.h>
31
32 using std::literals::chrono_literals::operator""ms;
33
34 namespace android {
35
36 /**
37 * Log debug messages about velocity tracking.
38 * Enable this via "adb shell setprop log.tag.VelocityTrackerVelocity DEBUG" (requires restart)
39 */
40 const bool DEBUG_VELOCITY =
41 __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Velocity", ANDROID_LOG_INFO);
42
43 /**
44 * Log debug messages about the progress of the algorithm itself.
45 * Enable this via "adb shell setprop log.tag.VelocityTrackerStrategy DEBUG" (requires restart)
46 */
47 const bool DEBUG_STRATEGY =
48 __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Strategy", ANDROID_LOG_INFO);
49
50 /**
51 * Log debug messages about the 'impulse' strategy.
52 * Enable this via "adb shell setprop log.tag.VelocityTrackerImpulse DEBUG" (requires restart)
53 */
54 const bool DEBUG_IMPULSE =
55 __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Impulse", ANDROID_LOG_INFO);
56
57 // Nanoseconds per milliseconds.
58 static const nsecs_t NANOS_PER_MS = 1000000;
59
60 // Seconds per nanosecond.
61 static const float SECONDS_PER_NANO = 1E-9;
62
63 // All axes supported for velocity tracking, mapped to their default strategies.
64 // Although other strategies are available for testing and comparison purposes,
65 // the default strategy is the one that applications will actually use. Be very careful
66 // when adjusting the default strategy because it can dramatically affect
67 // (often in a bad way) the user experience.
68 static const std::map<int32_t, VelocityTracker::Strategy> DEFAULT_STRATEGY_BY_AXIS =
69 {{AMOTION_EVENT_AXIS_X, VelocityTracker::Strategy::LSQ2},
70 {AMOTION_EVENT_AXIS_Y, VelocityTracker::Strategy::LSQ2},
71 {AMOTION_EVENT_AXIS_SCROLL, VelocityTracker::Strategy::IMPULSE}};
72
73 // Axes specifying location on a 2D plane (i.e. X and Y).
74 static const std::set<int32_t> PLANAR_AXES = {AMOTION_EVENT_AXIS_X, AMOTION_EVENT_AXIS_Y};
75
76 // Axes whose motion values are differential values (i.e. deltas).
77 static const std::set<int32_t> DIFFERENTIAL_AXES = {AMOTION_EVENT_AXIS_SCROLL};
78
79 // Threshold for determining that a pointer has stopped moving.
80 // Some input devices do not send ACTION_MOVE events in the case where a pointer has
81 // stopped. We need to detect this case so that we can accurately predict the
82 // velocity after the pointer starts moving again.
83 static const std::chrono::duration ASSUME_POINTER_STOPPED_TIME = 40ms;
84
toString(std::chrono::nanoseconds t)85 static std::string toString(std::chrono::nanoseconds t) {
86 std::stringstream stream;
87 stream.precision(1);
88 stream << std::fixed << std::chrono::duration<float, std::milli>(t).count() << " ms";
89 return stream.str();
90 }
91
vectorDot(const float * a,const float * b,uint32_t m)92 static float vectorDot(const float* a, const float* b, uint32_t m) {
93 float r = 0;
94 for (size_t i = 0; i < m; i++) {
95 r += *(a++) * *(b++);
96 }
97 return r;
98 }
99
vectorNorm(const float * a,uint32_t m)100 static float vectorNorm(const float* a, uint32_t m) {
101 float r = 0;
102 for (size_t i = 0; i < m; i++) {
103 float t = *(a++);
104 r += t * t;
105 }
106 return sqrtf(r);
107 }
108
vectorToString(const float * a,uint32_t m)109 static std::string vectorToString(const float* a, uint32_t m) {
110 std::string str;
111 str += "[";
112 for (size_t i = 0; i < m; i++) {
113 if (i) {
114 str += ",";
115 }
116 str += android::base::StringPrintf(" %f", *(a++));
117 }
118 str += " ]";
119 return str;
120 }
121
vectorToString(const std::vector<float> & v)122 static std::string vectorToString(const std::vector<float>& v) {
123 return vectorToString(v.data(), v.size());
124 }
125
matrixToString(const float * a,uint32_t m,uint32_t n,bool rowMajor)126 static std::string matrixToString(const float* a, uint32_t m, uint32_t n, bool rowMajor) {
127 std::string str;
128 str = "[";
129 for (size_t i = 0; i < m; i++) {
130 if (i) {
131 str += ",";
132 }
133 str += " [";
134 for (size_t j = 0; j < n; j++) {
135 if (j) {
136 str += ",";
137 }
138 str += android::base::StringPrintf(" %f", a[rowMajor ? i * n + j : j * m + i]);
139 }
140 str += " ]";
141 }
142 str += " ]";
143 return str;
144 }
145
146
147 // --- VelocityTracker ---
148
VelocityTracker(const Strategy strategy)149 VelocityTracker::VelocityTracker(const Strategy strategy)
150 : mLastEventTime(0), mCurrentPointerIdBits(0), mOverrideStrategy(strategy) {}
151
isAxisSupported(int32_t axis)152 bool VelocityTracker::isAxisSupported(int32_t axis) {
153 return DEFAULT_STRATEGY_BY_AXIS.find(axis) != DEFAULT_STRATEGY_BY_AXIS.end();
154 }
155
configureStrategy(int32_t axis)156 void VelocityTracker::configureStrategy(int32_t axis) {
157 const bool isDifferentialAxis = DIFFERENTIAL_AXES.find(axis) != DIFFERENTIAL_AXES.end();
158 if (isDifferentialAxis || mOverrideStrategy == VelocityTracker::Strategy::DEFAULT) {
159 // Do not allow overrides of strategies for differential axes, for now.
160 mConfiguredStrategies[axis] = createStrategy(DEFAULT_STRATEGY_BY_AXIS.at(axis),
161 /*deltaValues=*/isDifferentialAxis);
162 } else {
163 mConfiguredStrategies[axis] = createStrategy(mOverrideStrategy, /*deltaValues=*/false);
164 }
165 }
166
createStrategy(VelocityTracker::Strategy strategy,bool deltaValues)167 std::unique_ptr<VelocityTrackerStrategy> VelocityTracker::createStrategy(
168 VelocityTracker::Strategy strategy, bool deltaValues) {
169 switch (strategy) {
170 case VelocityTracker::Strategy::IMPULSE:
171 ALOGI_IF(DEBUG_STRATEGY, "Initializing impulse strategy");
172 return std::make_unique<ImpulseVelocityTrackerStrategy>(deltaValues);
173
174 case VelocityTracker::Strategy::LSQ1:
175 return std::make_unique<LeastSquaresVelocityTrackerStrategy>(1);
176
177 case VelocityTracker::Strategy::LSQ2:
178 ALOGI_IF(DEBUG_STRATEGY && !DEBUG_IMPULSE, "Initializing lsq2 strategy");
179 return std::make_unique<LeastSquaresVelocityTrackerStrategy>(2);
180
181 case VelocityTracker::Strategy::LSQ3:
182 return std::make_unique<LeastSquaresVelocityTrackerStrategy>(3);
183
184 case VelocityTracker::Strategy::WLSQ2_DELTA:
185 return std::make_unique<
186 LeastSquaresVelocityTrackerStrategy>(2,
187 LeastSquaresVelocityTrackerStrategy::
188 Weighting::DELTA);
189 case VelocityTracker::Strategy::WLSQ2_CENTRAL:
190 return std::make_unique<
191 LeastSquaresVelocityTrackerStrategy>(2,
192 LeastSquaresVelocityTrackerStrategy::
193 Weighting::CENTRAL);
194 case VelocityTracker::Strategy::WLSQ2_RECENT:
195 return std::make_unique<
196 LeastSquaresVelocityTrackerStrategy>(2,
197 LeastSquaresVelocityTrackerStrategy::
198 Weighting::RECENT);
199
200 case VelocityTracker::Strategy::INT1:
201 return std::make_unique<IntegratingVelocityTrackerStrategy>(1);
202
203 case VelocityTracker::Strategy::INT2:
204 return std::make_unique<IntegratingVelocityTrackerStrategy>(2);
205
206 case VelocityTracker::Strategy::LEGACY:
207 return std::make_unique<LegacyVelocityTrackerStrategy>();
208
209 default:
210 break;
211 }
212 LOG(FATAL) << "Invalid strategy: " << ftl::enum_string(strategy)
213 << ", deltaValues = " << deltaValues;
214
215 return nullptr;
216 }
217
clear()218 void VelocityTracker::clear() {
219 mCurrentPointerIdBits.clear();
220 mActivePointerId = std::nullopt;
221 mConfiguredStrategies.clear();
222 }
223
clearPointer(int32_t pointerId)224 void VelocityTracker::clearPointer(int32_t pointerId) {
225 mCurrentPointerIdBits.clearBit(pointerId);
226
227 if (mActivePointerId && *mActivePointerId == pointerId) {
228 // The active pointer id is being removed. Mark it invalid and try to find a new one
229 // from the remaining pointers.
230 mActivePointerId = std::nullopt;
231 if (!mCurrentPointerIdBits.isEmpty()) {
232 mActivePointerId = mCurrentPointerIdBits.firstMarkedBit();
233 }
234 }
235
236 for (const auto& [_, strategy] : mConfiguredStrategies) {
237 strategy->clearPointer(pointerId);
238 }
239 }
240
addMovement(nsecs_t eventTime,int32_t pointerId,int32_t axis,float position)241 void VelocityTracker::addMovement(nsecs_t eventTime, int32_t pointerId, int32_t axis,
242 float position) {
243 if (pointerId < 0 || pointerId > MAX_POINTER_ID) {
244 LOG(FATAL) << "Invalid pointer ID " << pointerId << " for axis "
245 << MotionEvent::getLabel(axis);
246 }
247
248 if (mCurrentPointerIdBits.hasBit(pointerId) &&
249 std::chrono::nanoseconds(eventTime - mLastEventTime) > ASSUME_POINTER_STOPPED_TIME) {
250 ALOGD_IF(DEBUG_VELOCITY, "VelocityTracker: stopped for %s, clearing state.",
251 toString(std::chrono::nanoseconds(eventTime - mLastEventTime)).c_str());
252
253 // We have not received any movements for too long. Assume that all pointers
254 // have stopped.
255 mConfiguredStrategies.clear();
256 }
257 mLastEventTime = eventTime;
258
259 mCurrentPointerIdBits.markBit(pointerId);
260 if (!mActivePointerId) {
261 // Let this be the new active pointer if no active pointer is currently set
262 mActivePointerId = pointerId;
263 }
264
265 if (mConfiguredStrategies.find(axis) == mConfiguredStrategies.end()) {
266 configureStrategy(axis);
267 }
268 mConfiguredStrategies[axis]->addMovement(eventTime, pointerId, position);
269
270 if (DEBUG_VELOCITY) {
271 LOG(INFO) << "VelocityTracker: addMovement axis=" << MotionEvent::getLabel(axis)
272 << ", eventTime=" << eventTime << ", pointerId=" << pointerId
273 << ", activePointerId=" << toString(mActivePointerId) << ", position=" << position
274 << ", velocity=" << toString(getVelocity(axis, pointerId));
275 }
276 }
277
addMovement(const MotionEvent & event)278 void VelocityTracker::addMovement(const MotionEvent& event) {
279 // Stores data about which axes to process based on the incoming motion event.
280 std::set<int32_t> axesToProcess;
281 int32_t actionMasked = event.getActionMasked();
282
283 switch (actionMasked) {
284 case AMOTION_EVENT_ACTION_DOWN:
285 case AMOTION_EVENT_ACTION_HOVER_ENTER:
286 // Clear all pointers on down before adding the new movement.
287 clear();
288 axesToProcess.insert(PLANAR_AXES.begin(), PLANAR_AXES.end());
289 break;
290 case AMOTION_EVENT_ACTION_POINTER_DOWN: {
291 // Start a new movement trace for a pointer that just went down.
292 // We do this on down instead of on up because the client may want to query the
293 // final velocity for a pointer that just went up.
294 clearPointer(event.getPointerId(event.getActionIndex()));
295 axesToProcess.insert(PLANAR_AXES.begin(), PLANAR_AXES.end());
296 break;
297 }
298 case AMOTION_EVENT_ACTION_MOVE:
299 case AMOTION_EVENT_ACTION_HOVER_MOVE:
300 axesToProcess.insert(PLANAR_AXES.begin(), PLANAR_AXES.end());
301 break;
302 case AMOTION_EVENT_ACTION_POINTER_UP:
303 if (event.getFlags() & AMOTION_EVENT_FLAG_CANCELED) {
304 clearPointer(event.getPointerId(event.getActionIndex()));
305 return;
306 }
307 // Continue to ACTION_UP to ensure that the POINTER_STOPPED logic is triggered.
308 [[fallthrough]];
309 case AMOTION_EVENT_ACTION_UP: {
310 std::chrono::nanoseconds delaySinceLastEvent(event.getEventTime() - mLastEventTime);
311 if (delaySinceLastEvent > ASSUME_POINTER_STOPPED_TIME) {
312 ALOGD_IF(DEBUG_VELOCITY,
313 "VelocityTracker: stopped for %s, clearing state upon pointer liftoff.",
314 toString(delaySinceLastEvent).c_str());
315 // We have not received any movements for too long. Assume that all pointers
316 // have stopped.
317 for (int32_t axis : PLANAR_AXES) {
318 mConfiguredStrategies.erase(axis);
319 }
320 }
321 // These actions because they do not convey any new information about
322 // pointer movement. We also want to preserve the last known velocity of the pointers.
323 // Note that ACTION_UP and ACTION_POINTER_UP always report the last known position
324 // of the pointers that went up. ACTION_POINTER_UP does include the new position of
325 // pointers that remained down but we will also receive an ACTION_MOVE with this
326 // information if any of them actually moved. Since we don't know how many pointers
327 // will be going up at once it makes sense to just wait for the following ACTION_MOVE
328 // before adding the movement.
329 return;
330 }
331 case AMOTION_EVENT_ACTION_SCROLL:
332 axesToProcess.insert(AMOTION_EVENT_AXIS_SCROLL);
333 break;
334 case AMOTION_EVENT_ACTION_CANCEL: {
335 clear();
336 return;
337 }
338
339 default:
340 // Ignore all other actions.
341 return;
342 }
343
344 const size_t historySize = event.getHistorySize();
345 for (size_t h = 0; h <= historySize; h++) {
346 const nsecs_t eventTime = event.getHistoricalEventTime(h);
347 for (size_t i = 0; i < event.getPointerCount(); i++) {
348 if (event.isResampled(i, h)) {
349 continue; // skip resampled samples
350 }
351 const int32_t pointerId = event.getPointerId(i);
352 for (int32_t axis : axesToProcess) {
353 const float position = event.getHistoricalAxisValue(axis, i, h);
354 addMovement(eventTime, pointerId, axis, position);
355 }
356 }
357 }
358 }
359
getVelocity(int32_t axis,int32_t pointerId) const360 std::optional<float> VelocityTracker::getVelocity(int32_t axis, int32_t pointerId) const {
361 const auto& it = mConfiguredStrategies.find(axis);
362 if (it != mConfiguredStrategies.end()) {
363 return it->second->getVelocity(pointerId);
364 }
365 return {};
366 }
367
getComputedVelocity(int32_t units,float maxVelocity)368 VelocityTracker::ComputedVelocity VelocityTracker::getComputedVelocity(int32_t units,
369 float maxVelocity) {
370 ComputedVelocity computedVelocity;
371 for (const auto& [axis, _] : mConfiguredStrategies) {
372 BitSet32 copyIdBits = BitSet32(mCurrentPointerIdBits);
373 while (!copyIdBits.isEmpty()) {
374 uint32_t id = copyIdBits.clearFirstMarkedBit();
375 std::optional<float> velocity = getVelocity(axis, id);
376 if (velocity) {
377 float adjustedVelocity =
378 std::clamp(*velocity * units / 1000, -maxVelocity, maxVelocity);
379 computedVelocity.addVelocity(axis, id, adjustedVelocity);
380 }
381 }
382 }
383 return computedVelocity;
384 }
385
AccumulatingVelocityTrackerStrategy(nsecs_t horizonNanos,bool maintainHorizonDuringAdd)386 AccumulatingVelocityTrackerStrategy::AccumulatingVelocityTrackerStrategy(
387 nsecs_t horizonNanos, bool maintainHorizonDuringAdd)
388 : mHorizonNanos(horizonNanos), mMaintainHorizonDuringAdd(maintainHorizonDuringAdd) {}
389
clearPointer(int32_t pointerId)390 void AccumulatingVelocityTrackerStrategy::clearPointer(int32_t pointerId) {
391 mMovements.erase(pointerId);
392 }
393
addMovement(nsecs_t eventTime,int32_t pointerId,float position)394 void AccumulatingVelocityTrackerStrategy::addMovement(nsecs_t eventTime, int32_t pointerId,
395 float position) {
396 auto [ringBufferIt, _] = mMovements.try_emplace(pointerId, HISTORY_SIZE);
397 RingBuffer<Movement>& movements = ringBufferIt->second;
398 const size_t size = movements.size();
399
400 if (size != 0 && movements[size - 1].eventTime == eventTime) {
401 // When ACTION_POINTER_DOWN happens, we will first receive ACTION_MOVE with the coordinates
402 // of the existing pointers, and then ACTION_POINTER_DOWN with the coordinates that include
403 // the new pointer. If the eventtimes for both events are identical, just update the data
404 // for this time (i.e. pop out the last element, and insert the updated movement).
405 // We only compare against the last value, as it is likely that addMovement is called
406 // in chronological order as events occur.
407 movements.popBack();
408 }
409
410 movements.pushBack({eventTime, position});
411
412 // Clear movements that do not fall within `mHorizonNanos` of the latest movement.
413 // Note that, if in the future we decide to use more movements (i.e. increase HISTORY_SIZE),
414 // we can consider making this step binary-search based, which will give us some improvement.
415 if (mMaintainHorizonDuringAdd) {
416 while (eventTime - movements[0].eventTime > mHorizonNanos) {
417 movements.popFront();
418 }
419 }
420 }
421
422 // --- LeastSquaresVelocityTrackerStrategy ---
423
LeastSquaresVelocityTrackerStrategy(uint32_t degree,Weighting weighting)424 LeastSquaresVelocityTrackerStrategy::LeastSquaresVelocityTrackerStrategy(uint32_t degree,
425 Weighting weighting)
426 : AccumulatingVelocityTrackerStrategy(HORIZON /*horizonNanos*/,
427 true /*maintainHorizonDuringAdd*/),
428 mDegree(degree),
429 mWeighting(weighting) {}
430
~LeastSquaresVelocityTrackerStrategy()431 LeastSquaresVelocityTrackerStrategy::~LeastSquaresVelocityTrackerStrategy() {}
432
433 /**
434 * Solves a linear least squares problem to obtain a N degree polynomial that fits
435 * the specified input data as nearly as possible.
436 *
437 * Returns true if a solution is found, false otherwise.
438 *
439 * The input consists of two vectors of data points X and Y with indices 0..m-1
440 * along with a weight vector W of the same size.
441 *
442 * The output is a vector B with indices 0..n that describes a polynomial
443 * that fits the data, such the sum of W[i] * W[i] * abs(Y[i] - (B[0] + B[1] X[i]
444 * + B[2] X[i]^2 ... B[n] X[i]^n)) for all i between 0 and m-1 is minimized.
445 *
446 * Accordingly, the weight vector W should be initialized by the caller with the
447 * reciprocal square root of the variance of the error in each input data point.
448 * In other words, an ideal choice for W would be W[i] = 1 / var(Y[i]) = 1 / stddev(Y[i]).
449 * The weights express the relative importance of each data point. If the weights are
450 * all 1, then the data points are considered to be of equal importance when fitting
451 * the polynomial. It is a good idea to choose weights that diminish the importance
452 * of data points that may have higher than usual error margins.
453 *
454 * Errors among data points are assumed to be independent. W is represented here
455 * as a vector although in the literature it is typically taken to be a diagonal matrix.
456 *
457 * That is to say, the function that generated the input data can be approximated
458 * by y(x) ~= B[0] + B[1] x + B[2] x^2 + ... + B[n] x^n.
459 *
460 * The coefficient of determination (R^2) is also returned to describe the goodness
461 * of fit of the model for the given data. It is a value between 0 and 1, where 1
462 * indicates perfect correspondence.
463 *
464 * This function first expands the X vector to a m by n matrix A such that
465 * A[i][0] = 1, A[i][1] = X[i], A[i][2] = X[i]^2, ..., A[i][n] = X[i]^n, then
466 * multiplies it by w[i]./
467 *
468 * Then it calculates the QR decomposition of A yielding an m by m orthonormal matrix Q
469 * and an m by n upper triangular matrix R. Because R is upper triangular (lower
470 * part is all zeroes), we can simplify the decomposition into an m by n matrix
471 * Q1 and a n by n matrix R1 such that A = Q1 R1.
472 *
473 * Finally we solve the system of linear equations given by R1 B = (Qtranspose W Y)
474 * to find B.
475 *
476 * For efficiency, we lay out A and Q column-wise in memory because we frequently
477 * operate on the column vectors. Conversely, we lay out R row-wise.
478 *
479 * http://en.wikipedia.org/wiki/Numerical_methods_for_linear_least_squares
480 * http://en.wikipedia.org/wiki/Gram-Schmidt
481 */
solveLeastSquares(const std::vector<float> & x,const std::vector<float> & y,const std::vector<float> & w,uint32_t n)482 static std::optional<float> solveLeastSquares(const std::vector<float>& x,
483 const std::vector<float>& y,
484 const std::vector<float>& w, uint32_t n) {
485 const size_t m = x.size();
486
487 ALOGD_IF(DEBUG_STRATEGY, "solveLeastSquares: m=%d, n=%d, x=%s, y=%s, w=%s", int(m), int(n),
488 vectorToString(x).c_str(), vectorToString(y).c_str(), vectorToString(w).c_str());
489
490 LOG_ALWAYS_FATAL_IF(m != y.size() || m != w.size(), "Mismatched vector sizes");
491
492 // Expand the X vector to a matrix A, pre-multiplied by the weights.
493 float a[n][m]; // column-major order
494 for (uint32_t h = 0; h < m; h++) {
495 a[0][h] = w[h];
496 for (uint32_t i = 1; i < n; i++) {
497 a[i][h] = a[i - 1][h] * x[h];
498 }
499 }
500
501 ALOGD_IF(DEBUG_STRATEGY, " - a=%s",
502 matrixToString(&a[0][0], m, n, /*rowMajor=*/false).c_str());
503
504 // Apply the Gram-Schmidt process to A to obtain its QR decomposition.
505 float q[n][m]; // orthonormal basis, column-major order
506 float r[n][n]; // upper triangular matrix, row-major order
507 for (uint32_t j = 0; j < n; j++) {
508 for (uint32_t h = 0; h < m; h++) {
509 q[j][h] = a[j][h];
510 }
511 for (uint32_t i = 0; i < j; i++) {
512 float dot = vectorDot(&q[j][0], &q[i][0], m);
513 for (uint32_t h = 0; h < m; h++) {
514 q[j][h] -= dot * q[i][h];
515 }
516 }
517
518 float norm = vectorNorm(&q[j][0], m);
519 if (norm < 0.000001f) {
520 // vectors are linearly dependent or zero so no solution
521 ALOGD_IF(DEBUG_STRATEGY, " - no solution, norm=%f", norm);
522 return {};
523 }
524
525 float invNorm = 1.0f / norm;
526 for (uint32_t h = 0; h < m; h++) {
527 q[j][h] *= invNorm;
528 }
529 for (uint32_t i = 0; i < n; i++) {
530 r[j][i] = i < j ? 0 : vectorDot(&q[j][0], &a[i][0], m);
531 }
532 }
533 if (DEBUG_STRATEGY) {
534 ALOGD(" - q=%s", matrixToString(&q[0][0], m, n, /*rowMajor=*/false).c_str());
535 ALOGD(" - r=%s", matrixToString(&r[0][0], n, n, /*rowMajor=*/true).c_str());
536
537 // calculate QR, if we factored A correctly then QR should equal A
538 float qr[n][m];
539 for (uint32_t h = 0; h < m; h++) {
540 for (uint32_t i = 0; i < n; i++) {
541 qr[i][h] = 0;
542 for (uint32_t j = 0; j < n; j++) {
543 qr[i][h] += q[j][h] * r[j][i];
544 }
545 }
546 }
547 ALOGD(" - qr=%s", matrixToString(&qr[0][0], m, n, /*rowMajor=*/false).c_str());
548 }
549
550 // Solve R B = Qt W Y to find B. This is easy because R is upper triangular.
551 // We just work from bottom-right to top-left calculating B's coefficients.
552 float wy[m];
553 for (uint32_t h = 0; h < m; h++) {
554 wy[h] = y[h] * w[h];
555 }
556 std::array<float, VelocityTracker::MAX_DEGREE + 1> outB;
557 for (uint32_t i = n; i != 0; ) {
558 i--;
559 outB[i] = vectorDot(&q[i][0], wy, m);
560 for (uint32_t j = n - 1; j > i; j--) {
561 outB[i] -= r[i][j] * outB[j];
562 }
563 outB[i] /= r[i][i];
564 }
565
566 ALOGD_IF(DEBUG_STRATEGY, " - b=%s", vectorToString(outB.data(), n).c_str());
567
568 // Calculate the coefficient of determination as 1 - (SSerr / SStot) where
569 // SSerr is the residual sum of squares (variance of the error),
570 // and SStot is the total sum of squares (variance of the data) where each
571 // has been weighted.
572 float ymean = 0;
573 for (uint32_t h = 0; h < m; h++) {
574 ymean += y[h];
575 }
576 ymean /= m;
577
578 if (DEBUG_STRATEGY) {
579 float sserr = 0;
580 float sstot = 0;
581 for (uint32_t h = 0; h < m; h++) {
582 float err = y[h] - outB[0];
583 float term = 1;
584 for (uint32_t i = 1; i < n; i++) {
585 term *= x[h];
586 err -= term * outB[i];
587 }
588 sserr += w[h] * w[h] * err * err;
589 float var = y[h] - ymean;
590 sstot += w[h] * w[h] * var * var;
591 }
592 ALOGD(" - sserr=%f", sserr);
593 ALOGD(" - sstot=%f", sstot);
594 }
595
596 return outB[1];
597 }
598
599 /*
600 * Optimized unweighted second-order least squares fit. About 2x speed improvement compared to
601 * the default implementation
602 */
solveUnweightedLeastSquaresDeg2(const RingBuffer<Movement> & movements) const603 std::optional<float> LeastSquaresVelocityTrackerStrategy::solveUnweightedLeastSquaresDeg2(
604 const RingBuffer<Movement>& movements) const {
605 // Solving y = a*x^2 + b*x + c, where
606 // - "x" is age (i.e. duration since latest movement) of the movemnets
607 // - "y" is positions of the movements.
608 float sxi = 0, sxiyi = 0, syi = 0, sxi2 = 0, sxi3 = 0, sxi2yi = 0, sxi4 = 0;
609
610 const size_t count = movements.size();
611 const Movement& newestMovement = movements[count - 1];
612 for (size_t i = 0; i < count; i++) {
613 const Movement& movement = movements[i];
614 nsecs_t age = newestMovement.eventTime - movement.eventTime;
615 float xi = -age * SECONDS_PER_NANO;
616 float yi = movement.position;
617
618 float xi2 = xi*xi;
619 float xi3 = xi2*xi;
620 float xi4 = xi3*xi;
621 float xiyi = xi*yi;
622 float xi2yi = xi2*yi;
623
624 sxi += xi;
625 sxi2 += xi2;
626 sxiyi += xiyi;
627 sxi2yi += xi2yi;
628 syi += yi;
629 sxi3 += xi3;
630 sxi4 += xi4;
631 }
632
633 float Sxx = sxi2 - sxi*sxi / count;
634 float Sxy = sxiyi - sxi*syi / count;
635 float Sxx2 = sxi3 - sxi*sxi2 / count;
636 float Sx2y = sxi2yi - sxi2*syi / count;
637 float Sx2x2 = sxi4 - sxi2*sxi2 / count;
638
639 float denominator = Sxx*Sx2x2 - Sxx2*Sxx2;
640 if (denominator == 0) {
641 ALOGW("division by 0 when computing velocity, Sxx=%f, Sx2x2=%f, Sxx2=%f", Sxx, Sx2x2, Sxx2);
642 return std::nullopt;
643 }
644
645 return (Sxy * Sx2x2 - Sx2y * Sxx2) / denominator;
646 }
647
getVelocity(int32_t pointerId) const648 std::optional<float> LeastSquaresVelocityTrackerStrategy::getVelocity(int32_t pointerId) const {
649 const auto movementIt = mMovements.find(pointerId);
650 if (movementIt == mMovements.end()) {
651 return std::nullopt; // no data
652 }
653
654 const RingBuffer<Movement>& movements = movementIt->second;
655 const size_t size = movements.size();
656 if (size == 0) {
657 return std::nullopt; // no data
658 }
659
660 uint32_t degree = mDegree;
661 if (degree > size - 1) {
662 degree = size - 1;
663 }
664
665 if (degree <= 0) {
666 return std::nullopt;
667 }
668
669 if (degree == 2 && mWeighting == Weighting::NONE) {
670 // Optimize unweighted, quadratic polynomial fit
671 return solveUnweightedLeastSquaresDeg2(movements);
672 }
673
674 // Iterate over movement samples in reverse time order and collect samples.
675 std::vector<float> positions;
676 std::vector<float> w;
677 std::vector<float> time;
678
679 const Movement& newestMovement = movements[size - 1];
680 for (ssize_t i = size - 1; i >= 0; i--) {
681 const Movement& movement = movements[i];
682 nsecs_t age = newestMovement.eventTime - movement.eventTime;
683 positions.push_back(movement.position);
684 w.push_back(chooseWeight(pointerId, i));
685 time.push_back(-age * 0.000000001f);
686 }
687
688 // General case for an Nth degree polynomial fit
689 return solveLeastSquares(time, positions, w, degree + 1);
690 }
691
chooseWeight(int32_t pointerId,uint32_t index) const692 float LeastSquaresVelocityTrackerStrategy::chooseWeight(int32_t pointerId, uint32_t index) const {
693 const RingBuffer<Movement>& movements = mMovements.at(pointerId);
694 const size_t size = movements.size();
695 switch (mWeighting) {
696 case Weighting::DELTA: {
697 // Weight points based on how much time elapsed between them and the next
698 // point so that points that "cover" a shorter time span are weighed less.
699 // delta 0ms: 0.5
700 // delta 10ms: 1.0
701 if (index == size - 1) {
702 return 1.0f;
703 }
704 float deltaMillis =
705 (movements[index + 1].eventTime - movements[index].eventTime) * 0.000001f;
706 if (deltaMillis < 0) {
707 return 0.5f;
708 }
709 if (deltaMillis < 10) {
710 return 0.5f + deltaMillis * 0.05;
711 }
712 return 1.0f;
713 }
714
715 case Weighting::CENTRAL: {
716 // Weight points based on their age, weighing very recent and very old points less.
717 // age 0ms: 0.5
718 // age 10ms: 1.0
719 // age 50ms: 1.0
720 // age 60ms: 0.5
721 float ageMillis =
722 (movements[size - 1].eventTime - movements[index].eventTime) * 0.000001f;
723 if (ageMillis < 0) {
724 return 0.5f;
725 }
726 if (ageMillis < 10) {
727 return 0.5f + ageMillis * 0.05;
728 }
729 if (ageMillis < 50) {
730 return 1.0f;
731 }
732 if (ageMillis < 60) {
733 return 0.5f + (60 - ageMillis) * 0.05;
734 }
735 return 0.5f;
736 }
737
738 case Weighting::RECENT: {
739 // Weight points based on their age, weighing older points less.
740 // age 0ms: 1.0
741 // age 50ms: 1.0
742 // age 100ms: 0.5
743 float ageMillis =
744 (movements[size - 1].eventTime - movements[index].eventTime) * 0.000001f;
745 if (ageMillis < 50) {
746 return 1.0f;
747 }
748 if (ageMillis < 100) {
749 return 0.5f + (100 - ageMillis) * 0.01f;
750 }
751 return 0.5f;
752 }
753
754 case Weighting::NONE:
755 return 1.0f;
756 }
757 }
758
759 // --- IntegratingVelocityTrackerStrategy ---
760
IntegratingVelocityTrackerStrategy(uint32_t degree)761 IntegratingVelocityTrackerStrategy::IntegratingVelocityTrackerStrategy(uint32_t degree) :
762 mDegree(degree) {
763 }
764
~IntegratingVelocityTrackerStrategy()765 IntegratingVelocityTrackerStrategy::~IntegratingVelocityTrackerStrategy() {
766 }
767
clearPointer(int32_t pointerId)768 void IntegratingVelocityTrackerStrategy::clearPointer(int32_t pointerId) {
769 mPointerIdBits.clearBit(pointerId);
770 }
771
addMovement(nsecs_t eventTime,int32_t pointerId,float position)772 void IntegratingVelocityTrackerStrategy::addMovement(nsecs_t eventTime, int32_t pointerId,
773 float position) {
774 State& state = mPointerState[pointerId];
775 if (mPointerIdBits.hasBit(pointerId)) {
776 updateState(state, eventTime, position);
777 } else {
778 initState(state, eventTime, position);
779 }
780
781 mPointerIdBits.markBit(pointerId);
782 }
783
getVelocity(int32_t pointerId) const784 std::optional<float> IntegratingVelocityTrackerStrategy::getVelocity(int32_t pointerId) const {
785 if (mPointerIdBits.hasBit(pointerId)) {
786 return mPointerState[pointerId].vel;
787 }
788
789 return std::nullopt;
790 }
791
initState(State & state,nsecs_t eventTime,float pos) const792 void IntegratingVelocityTrackerStrategy::initState(State& state, nsecs_t eventTime,
793 float pos) const {
794 state.updateTime = eventTime;
795 state.degree = 0;
796
797 state.pos = pos;
798 state.accel = 0;
799 state.vel = 0;
800 }
801
updateState(State & state,nsecs_t eventTime,float pos) const802 void IntegratingVelocityTrackerStrategy::updateState(State& state, nsecs_t eventTime,
803 float pos) const {
804 const nsecs_t MIN_TIME_DELTA = 2 * NANOS_PER_MS;
805 const float FILTER_TIME_CONSTANT = 0.010f; // 10 milliseconds
806
807 if (eventTime <= state.updateTime + MIN_TIME_DELTA) {
808 return;
809 }
810
811 float dt = (eventTime - state.updateTime) * 0.000000001f;
812 state.updateTime = eventTime;
813
814 float vel = (pos - state.pos) / dt;
815 if (state.degree == 0) {
816 state.vel = vel;
817 state.degree = 1;
818 } else {
819 float alpha = dt / (FILTER_TIME_CONSTANT + dt);
820 if (mDegree == 1) {
821 state.vel += (vel - state.vel) * alpha;
822 } else {
823 float accel = (vel - state.vel) / dt;
824 if (state.degree == 1) {
825 state.accel = accel;
826 state.degree = 2;
827 } else {
828 state.accel += (accel - state.accel) * alpha;
829 }
830 state.vel += (state.accel * dt) * alpha;
831 }
832 }
833 state.pos = pos;
834 }
835
836 // --- LegacyVelocityTrackerStrategy ---
837
LegacyVelocityTrackerStrategy()838 LegacyVelocityTrackerStrategy::LegacyVelocityTrackerStrategy()
839 : AccumulatingVelocityTrackerStrategy(HORIZON /*horizonNanos*/,
840 false /*maintainHorizonDuringAdd*/) {}
841
~LegacyVelocityTrackerStrategy()842 LegacyVelocityTrackerStrategy::~LegacyVelocityTrackerStrategy() {
843 }
844
getVelocity(int32_t pointerId) const845 std::optional<float> LegacyVelocityTrackerStrategy::getVelocity(int32_t pointerId) const {
846 const auto movementIt = mMovements.find(pointerId);
847 if (movementIt == mMovements.end()) {
848 return std::nullopt; // no data
849 }
850
851 const RingBuffer<Movement>& movements = movementIt->second;
852 const size_t size = movements.size();
853 if (size == 0) {
854 return std::nullopt; // no data
855 }
856
857 const Movement& newestMovement = movements[size - 1];
858
859 // Find the oldest sample that contains the pointer and that is not older than HORIZON.
860 nsecs_t minTime = newestMovement.eventTime - HORIZON;
861 uint32_t oldestIndex = size - 1;
862 for (ssize_t i = size - 1; i >= 0; i--) {
863 const Movement& nextOldestMovement = movements[i];
864 if (nextOldestMovement.eventTime < minTime) {
865 break;
866 }
867 oldestIndex = i;
868 }
869
870 // Calculate an exponentially weighted moving average of the velocity estimate
871 // at different points in time measured relative to the oldest sample.
872 // This is essentially an IIR filter. Newer samples are weighted more heavily
873 // than older samples. Samples at equal time points are weighted more or less
874 // equally.
875 //
876 // One tricky problem is that the sample data may be poorly conditioned.
877 // Sometimes samples arrive very close together in time which can cause us to
878 // overestimate the velocity at that time point. Most samples might be measured
879 // 16ms apart but some consecutive samples could be only 0.5sm apart because
880 // the hardware or driver reports them irregularly or in bursts.
881 float accumV = 0;
882 uint32_t samplesUsed = 0;
883 const Movement& oldestMovement = movements[oldestIndex];
884 float oldestPosition = oldestMovement.position;
885 nsecs_t lastDuration = 0;
886
887 for (size_t i = oldestIndex; i < size; i++) {
888 const Movement& movement = movements[i];
889 nsecs_t duration = movement.eventTime - oldestMovement.eventTime;
890
891 // If the duration between samples is small, we may significantly overestimate
892 // the velocity. Consequently, we impose a minimum duration constraint on the
893 // samples that we include in the calculation.
894 if (duration >= MIN_DURATION) {
895 float position = movement.position;
896 float scale = 1000000000.0f / duration; // one over time delta in seconds
897 float v = (position - oldestPosition) * scale;
898 accumV = (accumV * lastDuration + v * duration) / (duration + lastDuration);
899 lastDuration = duration;
900 samplesUsed += 1;
901 }
902 }
903
904 if (samplesUsed) {
905 return accumV;
906 }
907 return std::nullopt;
908 }
909
910 // --- ImpulseVelocityTrackerStrategy ---
911
ImpulseVelocityTrackerStrategy(bool deltaValues)912 ImpulseVelocityTrackerStrategy::ImpulseVelocityTrackerStrategy(bool deltaValues)
913 : AccumulatingVelocityTrackerStrategy(HORIZON /*horizonNanos*/,
914 true /*maintainHorizonDuringAdd*/),
915 mDeltaValues(deltaValues) {}
916
~ImpulseVelocityTrackerStrategy()917 ImpulseVelocityTrackerStrategy::~ImpulseVelocityTrackerStrategy() {
918 }
919
920 /**
921 * Calculate the total impulse provided to the screen and the resulting velocity.
922 *
923 * The touchscreen is modeled as a physical object.
924 * Initial condition is discussed below, but for now suppose that v(t=0) = 0
925 *
926 * The kinetic energy of the object at the release is E=0.5*m*v^2
927 * Then vfinal = sqrt(2E/m). The goal is to calculate E.
928 *
929 * The kinetic energy at the release is equal to the total work done on the object by the finger.
930 * The total work W is the sum of all dW along the path.
931 *
932 * dW = F*dx, where dx is the piece of path traveled.
933 * Force is change of momentum over time, F = dp/dt = m dv/dt.
934 * Then substituting:
935 * dW = m (dv/dt) * dx = m * v * dv
936 *
937 * Summing along the path, we get:
938 * W = sum(dW) = sum(m * v * dv) = m * sum(v * dv)
939 * Since the mass stays constant, the equation for final velocity is:
940 * vfinal = sqrt(2*sum(v * dv))
941 *
942 * Here,
943 * dv : change of velocity = (v[i+1]-v[i])
944 * dx : change of distance = (x[i+1]-x[i])
945 * dt : change of time = (t[i+1]-t[i])
946 * v : instantaneous velocity = dx/dt
947 *
948 * The final formula is:
949 * vfinal = sqrt(2) * sqrt(sum((v[i]-v[i-1])*|v[i]|)) for all i
950 * The absolute value is needed to properly account for the sign. If the velocity over a
951 * particular segment descreases, then this indicates braking, which means that negative
952 * work was done. So for two positive, but decreasing, velocities, this contribution would be
953 * negative and will cause a smaller final velocity.
954 *
955 * Initial condition
956 * There are two ways to deal with initial condition:
957 * 1) Assume that v(0) = 0, which would mean that the screen is initially at rest.
958 * This is not entirely accurate. We are only taking the past X ms of touch data, where X is
959 * currently equal to 100. However, a touch event that created a fling probably lasted for longer
960 * than that, which would mean that the user has already been interacting with the touchscreen
961 * and it has probably already been moving.
962 * 2) Assume that the touchscreen has already been moving at a certain velocity, calculate this
963 * initial velocity and the equivalent energy, and start with this initial energy.
964 * Consider an example where we have the following data, consisting of 3 points:
965 * time: t0, t1, t2
966 * x : x0, x1, x2
967 * v : 0 , v1, v2
968 * Here is what will happen in each of these scenarios:
969 * 1) By directly applying the formula above with the v(0) = 0 boundary condition, we will get
970 * vfinal = sqrt(2*(|v1|*(v1-v0) + |v2|*(v2-v1))). This can be simplified since v0=0
971 * vfinal = sqrt(2*(|v1|*v1 + |v2|*(v2-v1))) = sqrt(2*(v1^2 + |v2|*(v2 - v1)))
972 * since velocity is a real number
973 * 2) If we treat the screen as already moving, then it must already have an energy (per mass)
974 * equal to 1/2*v1^2. Then the initial energy should be 1/2*v1*2, and only the second segment
975 * will contribute to the total kinetic energy (since we can effectively consider that v0=v1).
976 * This will give the following expression for the final velocity:
977 * vfinal = sqrt(2*(1/2*v1^2 + |v2|*(v2-v1)))
978 * This analysis can be generalized to an arbitrary number of samples.
979 *
980 *
981 * Comparing the two equations above, we see that the only mathematical difference
982 * is the factor of 1/2 in front of the first velocity term.
983 * This boundary condition would allow for the "proper" calculation of the case when all of the
984 * samples are equally spaced in time and distance, which should suggest a constant velocity.
985 *
986 * Note that approach 2) is sensitive to the proper ordering of the data in time, since
987 * the boundary condition must be applied to the oldest sample to be accurate.
988 */
kineticEnergyToVelocity(float work)989 static float kineticEnergyToVelocity(float work) {
990 static constexpr float sqrt2 = 1.41421356237;
991 return (work < 0 ? -1.0 : 1.0) * sqrtf(fabsf(work)) * sqrt2;
992 }
993
getVelocity(int32_t pointerId) const994 std::optional<float> ImpulseVelocityTrackerStrategy::getVelocity(int32_t pointerId) const {
995 const auto movementIt = mMovements.find(pointerId);
996 if (movementIt == mMovements.end()) {
997 return std::nullopt; // no data
998 }
999
1000 const RingBuffer<Movement>& movements = movementIt->second;
1001 const size_t size = movements.size();
1002 if (size == 0) {
1003 return std::nullopt; // no data
1004 }
1005
1006 float work = 0;
1007 for (size_t i = 0; i < size - 1; i++) {
1008 const Movement& mvt = movements[i];
1009 const Movement& nextMvt = movements[i + 1];
1010
1011 float vprev = kineticEnergyToVelocity(work);
1012 float delta = mDeltaValues ? nextMvt.position : nextMvt.position - mvt.position;
1013 float vcurr = delta / (SECONDS_PER_NANO * (nextMvt.eventTime - mvt.eventTime));
1014 work += (vcurr - vprev) * fabsf(vcurr);
1015
1016 if (i == 0) {
1017 work *= 0.5; // initial condition, case 2) above
1018 }
1019 }
1020
1021 const float velocity = kineticEnergyToVelocity(work);
1022 ALOGD_IF(DEBUG_STRATEGY, "velocity: %.1f", velocity);
1023
1024 if (DEBUG_IMPULSE) {
1025 // TODO(b/134179997): delete this block once the switch to 'impulse' is complete.
1026 // Calculate the lsq2 velocity for the same inputs to allow runtime comparisons.
1027 // X axis chosen arbitrarily for velocity comparisons.
1028 VelocityTracker lsq2(VelocityTracker::Strategy::LSQ2);
1029 for (size_t i = 0; i < size; i++) {
1030 const Movement& mvt = movements[i];
1031 lsq2.addMovement(mvt.eventTime, pointerId, AMOTION_EVENT_AXIS_X, mvt.position);
1032 }
1033 std::optional<float> v = lsq2.getVelocity(AMOTION_EVENT_AXIS_X, pointerId);
1034 if (v) {
1035 ALOGD("lsq2 velocity: %.1f", *v);
1036 } else {
1037 ALOGD("lsq2 velocity: could not compute velocity");
1038 }
1039 }
1040 return velocity;
1041 }
1042
1043 } // namespace android
1044