1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <android-base/thread_annotations.h>
20 #include <future>
21 #include <thread>
22 #include <unordered_map>
23 
24 #include <aidl/android/hardware/input/processor/IInputProcessor.h>
25 #include <input/BlockingQueue.h>
26 #include "InputListener.h"
27 namespace android {
28 
29 enum class ClassifierEventType : uint8_t {
30     MOTION = 0,
31     DEVICE_RESET = 1,
32     HAL_RESET = 2,
33     EXIT = 3,
34 };
35 
36 struct ClassifierEvent {
37     ClassifierEventType type;
38     std::optional<NotifyArgs> args;
39 
40     ClassifierEvent(ClassifierEventType type, std::optional<NotifyArgs> args);
41     ClassifierEvent(const NotifyMotionArgs& args);
42     ClassifierEvent(const NotifyDeviceResetArgs& args);
43     ClassifierEvent(ClassifierEvent&& other) = default;
44     ClassifierEvent& operator=(ClassifierEvent&& other);
45 
46     // Convenience function to create a HAL_RESET event
47     static ClassifierEvent createHalResetEvent();
48     // Convenience function to create an EXIT event
49     static ClassifierEvent createExitEvent();
50 
51     std::optional<int32_t> getDeviceId() const;
52 };
53 
54 // --- Interfaces ---
55 
56 /**
57  * Interface for adding a MotionClassification to NotifyMotionArgs.
58  *
59  * To implement, override the classify function.
60  */
61 class MotionClassifierInterface {
62 public:
MotionClassifierInterface()63     MotionClassifierInterface() {}
~MotionClassifierInterface()64     virtual ~MotionClassifierInterface() {}
65     /**
66      * Based on the motion event described by NotifyMotionArgs,
67      * provide a MotionClassification for the current gesture.
68      */
69     virtual MotionClassification classify(const NotifyMotionArgs& args) = 0;
70     /**
71      * Reset all internal HAL state.
72      */
73     virtual void reset() = 0;
74     /**
75      * Reset HAL state for a specific device.
76      */
77     virtual void reset(const NotifyDeviceResetArgs& args) = 0;
78 
79     /**
80      * Dump the state of the motion classifier.
81      */
82     virtual void dump(std::string& dump) = 0;
83 
84     /**
85      * Called by the heartbeat to ensure the HAL is still processing normally.
86      */
87     virtual void monitor() = 0;
88 };
89 
90 /**
91  * Base interface for an InputListener stage.
92  * Provides classification to events.
93  */
94 class InputProcessorInterface : public InputListenerInterface {
95 public:
96     virtual void setMotionClassifierEnabled(bool enabled) = 0;
97     /**
98      * Dump the state of the input classifier.
99      * This method may be called on any thread (usually by the input manager).
100      */
101     virtual void dump(std::string& dump) = 0;
102 
103     /** Called by the heartbeat to ensure that the classifier has not deadlocked. */
104     virtual void monitor() = 0;
105 
InputProcessorInterface()106     InputProcessorInterface() {}
~InputProcessorInterface()107     virtual ~InputProcessorInterface() {}
108 };
109 
110 // --- Implementations ---
111 
112 class ScopedDeathRecipient {
113 public:
114     explicit ScopedDeathRecipient(AIBinder_DeathRecipient_onBinderDied onBinderDied, void* cookie);
115     ScopedDeathRecipient(const ScopedDeathRecipient&) = delete;
116     ScopedDeathRecipient& operator=(ScopedDeathRecipient const&) = delete;
117     void linkToDeath(AIBinder* binder);
118     ~ScopedDeathRecipient();
119 
120 private:
121     AIBinder_DeathRecipient* mRecipient;
122     void* mCookie;
123 };
124 
125 /**
126  * Implementation of MotionClassifierInterface that calls the InputProcessor HAL
127  * in order to determine the classification for the current gesture.
128  *
129  * The InputProcessor HAL may keep track of the entire gesture in order to determine
130  * the classification, and may be hardware-specific. It may use the data in
131  * NotifyMotionArgs::videoFrames field to drive the classification decisions.
132  * The HAL is called from a separate thread.
133  */
134 class MotionClassifier final : public MotionClassifierInterface {
135 public:
136     /*
137      * Create an instance of MotionClassifier.
138      * The death recipient, if provided, will be subscribed to the HAL death.
139      * The death recipient could be used to destroy MotionClassifier.
140      *
141      * This function should be called asynchronously, because getService takes a long time.
142      */
143     static std::unique_ptr<MotionClassifierInterface> create(
144             std::shared_ptr<aidl::android::hardware::input::processor::IInputProcessor> service);
145 
146     ~MotionClassifier();
147 
148     /**
149      * Classifies events asynchronously; that is, it doesn't block events on a classification,
150      * but instead sends them over to the classifier HAL. After a classification of a specific
151      * event is determined, MotionClassifier then marks the next event in the stream with this
152      * classification.
153      *
154      * Therefore, it is acceptable to have the classifications be delayed by 1-2 events
155      * in a particular gesture.
156      */
157     virtual MotionClassification classify(const NotifyMotionArgs& args) override;
158     virtual void reset() override;
159     virtual void reset(const NotifyDeviceResetArgs& args) override;
160 
161     virtual void dump(std::string& dump) override;
162     virtual void monitor() override;
163 
164 private:
165     friend class MotionClassifierTest; // to create MotionClassifier with a test HAL implementation
166     explicit MotionClassifier(
167             std::shared_ptr<aidl::android::hardware::input::processor::IInputProcessor> service);
168 
169     /** The events that need to be sent to the HAL. */
170     BlockingQueue<ClassifierEvent> mEvents;
171     /**
172      * Add an event to the queue mEvents.
173      */
174     void enqueueEvent(ClassifierEvent&& event);
175     /**
176      * Thread that will communicate with InputProcessor HAL.
177      * This should be the only thread that communicates with InputProcessor HAL,
178      * because this thread is allowed to block on the HAL calls.
179      */
180     std::thread mHalThread;
181     /**
182      * Process events and call the InputProcessor HAL
183      */
184     void processEvents();
185     /**
186      * Access to the InputProcessor HAL. May be null if init() hasn't completed yet.
187      * When init() successfully completes, mService is guaranteed to remain non-null and to not
188      * change its value until MotionClassifier is destroyed.
189      * This variable is *not* guarded by mLock in the InputProcessor thread, because
190      * that thread knows exactly when this variable is initialized.
191      * When accessed in any other thread, mService is checked for nullness with a lock.
192      */
193     std::shared_ptr<aidl::android::hardware::input::processor::IInputProcessor> mService;
194     std::mutex mLock;
195     /**
196      * Per-device input classifications. Should only be accessed using the
197      * getClassification / setClassification methods.
198      */
199     std::unordered_map<int32_t /*deviceId*/, MotionClassification> mClassifications
200             GUARDED_BY(mLock);
201     /**
202      * Set the current classification for a given device.
203      */
204     void setClassification(int32_t deviceId, MotionClassification classification);
205     /**
206      * Get the current classification for a given device.
207      */
208     MotionClassification getClassification(int32_t deviceId);
209     void updateClassification(int32_t deviceId, nsecs_t eventTime,
210                               MotionClassification classification);
211     /**
212      * Clear all current classifications
213      */
214     void clearClassifications();
215     /**
216      * Per-device times when the last ACTION_DOWN was received.
217      * Used to reject late classifications that do not belong to the current gesture.
218      *
219      * Accessed indirectly by both InputProcessor thread and the thread that receives notifyMotion.
220      */
221     std::unordered_map<int32_t /*deviceId*/, nsecs_t /*downTime*/> mLastDownTimes GUARDED_BY(mLock);
222 
223     void updateLastDownTime(int32_t deviceId, nsecs_t downTime);
224 
225     void clearDeviceState(int32_t deviceId);
226 
227     /**
228      * Exit the InputProcessor HAL thread.
229      * Useful for tests to ensure proper cleanup.
230      */
231     void requestExit();
232     /**
233      * Return string status of mService
234      */
235     const char* getServiceStatus() REQUIRES(mLock);
236 };
237 
238 /**
239  * Implementation of the InputProcessorInterface.
240  * Represents a separate stage of input processing. All of the input events go through this stage.
241  * Acts as a passthrough for all input events except for motion events.
242  * The events of motion type are sent to MotionClassifier.
243  */
244 class InputProcessor : public InputProcessorInterface {
245 public:
246     explicit InputProcessor(InputListenerInterface& listener);
247 
248     void notifyInputDevicesChanged(const NotifyInputDevicesChangedArgs& args) override;
249     void notifyConfigurationChanged(const NotifyConfigurationChangedArgs& args) override;
250     void notifyKey(const NotifyKeyArgs& args) override;
251     void notifyMotion(const NotifyMotionArgs& args) override;
252     void notifySwitch(const NotifySwitchArgs& args) override;
253     void notifySensor(const NotifySensorArgs& args) override;
254     void notifyVibratorState(const NotifyVibratorStateArgs& args) override;
255     void notifyDeviceReset(const NotifyDeviceResetArgs& args) override;
256     void notifyPointerCaptureChanged(const NotifyPointerCaptureChangedArgs& args) override;
257 
258     void dump(std::string& dump) override;
259     void monitor() override;
260 
261     ~InputProcessor();
262 
263     // Called from InputManager
264     void setMotionClassifierEnabled(bool enabled) override;
265 
266 private:
267     // Protect access to mMotionClassifier, since it may become null via a hidl callback
268     std::mutex mLock;
269     // The next stage to pass input events to
270     QueuedInputListener mQueuedListener;
271 
272     std::unique_ptr<MotionClassifierInterface> mMotionClassifier GUARDED_BY(mLock);
273     std::future<void> mInitializeMotionClassifier GUARDED_BY(mLock);
274 
275     /**
276      * Set the value of mMotionClassifier.
277      * This is called from 2 different threads:
278      * 1) mInitializeMotionClassifierThread, when we have constructed a MotionClassifier
279      * 2) A binder thread of the HalDeathRecipient, which is created when HAL dies. This would cause
280      *    mMotionClassifier to become nullptr.
281      */
282     void setMotionClassifierLocked(std::unique_ptr<MotionClassifierInterface> motionClassifier)
283             REQUIRES(mLock);
284 
285     static void onBinderDied(void* cookie);
286 
287     std::unique_ptr<ScopedDeathRecipient> mHalDeathRecipient GUARDED_BY(mLock);
288 };
289 
290 } // namespace android
291