1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef _UI_INPUT_CLASSIFIER_H
18 #define _UI_INPUT_CLASSIFIER_H
19 
20 #include <android-base/thread_annotations.h>
21 #include <utils/RefBase.h>
22 #include <thread>
23 #include <unordered_map>
24 
25 #include "BlockingQueue.h"
26 #include "InputListener.h"
27 #include <android/hardware/input/classifier/1.0/IInputClassifier.h>
28 
29 namespace android {
30 
31 enum class ClassifierEventType : uint8_t {
32     MOTION = 0,
33     DEVICE_RESET = 1,
34     HAL_RESET = 2,
35     EXIT = 3,
36 };
37 
38 struct ClassifierEvent {
39     ClassifierEventType type;
40     std::unique_ptr<NotifyArgs> args;
41 
42     ClassifierEvent(ClassifierEventType type, std::unique_ptr<NotifyArgs> args);
43     ClassifierEvent(std::unique_ptr<NotifyMotionArgs> args);
44     ClassifierEvent(std::unique_ptr<NotifyDeviceResetArgs> args);
45     ClassifierEvent(ClassifierEvent&& other);
46     ClassifierEvent& operator=(ClassifierEvent&& other);
47 
48     // Convenience function to create a HAL_RESET event
49     static ClassifierEvent createHalResetEvent();
50     // Convenience function to create an EXIT event
51     static ClassifierEvent createExitEvent();
52 
53     std::optional<int32_t> getDeviceId() const;
54 };
55 
56 // --- Interfaces ---
57 
58 /**
59  * Interface for adding a MotionClassification to NotifyMotionArgs.
60  *
61  * To implement, override the classify function.
62  */
63 class MotionClassifierInterface {
64 public:
MotionClassifierInterface()65     MotionClassifierInterface() { }
~MotionClassifierInterface()66     virtual ~MotionClassifierInterface() { }
67     /**
68      * Based on the motion event described by NotifyMotionArgs,
69      * provide a MotionClassification for the current gesture.
70      */
71     virtual MotionClassification classify(const NotifyMotionArgs& args) = 0;
72     /**
73      * Reset all internal HAL state.
74      */
75     virtual void reset() = 0;
76     /**
77      * Reset HAL state for a specific device.
78      */
79     virtual void reset(const NotifyDeviceResetArgs& args) = 0;
80 
81     /**
82      * Dump the state of the motion classifier
83      */
84     virtual void dump(std::string& dump) = 0;
85 };
86 
87 /**
88  * Base interface for an InputListener stage.
89  * Provides classification to events.
90  */
91 class InputClassifierInterface : public virtual RefBase, public InputListenerInterface {
92 public:
93     virtual void setMotionClassifierEnabled(bool enabled) = 0;
94     /**
95      * Dump the state of the input classifier.
96      * This method may be called on any thread (usually by the input manager).
97      */
98     virtual void dump(std::string& dump) = 0;
99 protected:
InputClassifierInterface()100     InputClassifierInterface() { }
~InputClassifierInterface()101     virtual ~InputClassifierInterface() { }
102 };
103 
104 // --- Implementations ---
105 
106 /**
107  * Implementation of MotionClassifierInterface that calls the InputClassifier HAL
108  * in order to determine the classification for the current gesture.
109  *
110  * The InputClassifier HAL may keep track of the entire gesture in order to determine
111  * the classification, and may be hardware-specific. It may use the data in
112  * NotifyMotionArgs::videoFrames field to drive the classification decisions.
113  * The HAL is called from a separate thread.
114  */
115 class MotionClassifier final : public MotionClassifierInterface {
116 public:
117     /*
118      * Create an instance of MotionClassifier.
119      * The death recipient, if provided, will be subscribed to the HAL death.
120      * The death recipient could be used to destroy MotionClassifier.
121      *
122      * This function should be called asynchronously, because getService takes a long time.
123      */
124     static std::unique_ptr<MotionClassifierInterface> create(
125             sp<android::hardware::hidl_death_recipient> deathRecipient);
126 
127     ~MotionClassifier();
128 
129     /**
130      * Classifies events asynchronously; that is, it doesn't block events on a classification,
131      * but instead sends them over to the classifier HAL. After a classification of a specific
132      * event is determined, MotionClassifier then marks the next event in the stream with this
133      * classification.
134      *
135      * Therefore, it is acceptable to have the classifications be delayed by 1-2 events
136      * in a particular gesture.
137      */
138     virtual MotionClassification classify(const NotifyMotionArgs& args) override;
139     virtual void reset() override;
140     virtual void reset(const NotifyDeviceResetArgs& args) override;
141 
142     virtual void dump(std::string& dump) override;
143 
144 private:
145     friend class MotionClassifierTest; // to create MotionClassifier with a test HAL implementation
146     explicit MotionClassifier(
147             sp<android::hardware::input::classifier::V1_0::IInputClassifier> service);
148 
149     // The events that need to be sent to the HAL.
150     BlockingQueue<ClassifierEvent> mEvents;
151     /**
152      * Add an event to the queue mEvents.
153      */
154     void enqueueEvent(ClassifierEvent&& event);
155     /**
156      * Thread that will communicate with InputClassifier HAL.
157      * This should be the only thread that communicates with InputClassifier HAL,
158      * because this thread is allowed to block on the HAL calls.
159      */
160     std::thread mHalThread;
161     /**
162      * Process events and call the InputClassifier HAL
163      */
164     void processEvents();
165     /**
166      * Access to the InputClassifier HAL. May be null if init() hasn't completed yet.
167      * When init() successfully completes, mService is guaranteed to remain non-null and to not
168      * change its value until MotionClassifier is destroyed.
169      * This variable is *not* guarded by mLock in the InputClassifier thread, because
170      * that thread knows exactly when this variable is initialized.
171      * When accessed in any other thread, mService is checked for nullness with a lock.
172      */
173     sp<android::hardware::input::classifier::V1_0::IInputClassifier> mService;
174     std::mutex mLock;
175     /**
176      * Per-device input classifications. Should only be accessed using the
177      * getClassification / setClassification methods.
178      */
179     std::unordered_map<int32_t /*deviceId*/, MotionClassification>
180             mClassifications GUARDED_BY(mLock);
181     /**
182      * Set the current classification for a given device.
183      */
184     void setClassification(int32_t deviceId, MotionClassification classification);
185     /**
186      * Get the current classification for a given device.
187      */
188     MotionClassification getClassification(int32_t deviceId);
189     void updateClassification(int32_t deviceId, nsecs_t eventTime,
190             MotionClassification classification);
191     /**
192      * Clear all current classifications
193      */
194     void clearClassifications();
195     /**
196      * Per-device times when the last ACTION_DOWN was received.
197      * Used to reject late classifications that do not belong to the current gesture.
198      *
199      * Accessed indirectly by both InputClassifier thread and the thread that receives notifyMotion.
200      */
201     std::unordered_map<int32_t /*deviceId*/, nsecs_t /*downTime*/> mLastDownTimes GUARDED_BY(mLock);
202 
203     void updateLastDownTime(int32_t deviceId, nsecs_t downTime);
204 
205     void clearDeviceState(int32_t deviceId);
206 
207     /**
208      * Exit the InputClassifier HAL thread.
209      * Useful for tests to ensure proper cleanup.
210      */
211     void requestExit();
212     /**
213      * Return string status of mService
214      */
215     const char* getServiceStatus() REQUIRES(mLock);
216 };
217 
218 /**
219  * Implementation of the InputClassifierInterface.
220  * Represents a separate stage of input processing. All of the input events go through this stage.
221  * Acts as a passthrough for all input events except for motion events.
222  * The events of motion type are sent to MotionClassifier.
223  */
224 class InputClassifier : public InputClassifierInterface {
225 public:
226     explicit InputClassifier(const sp<InputListenerInterface>& listener);
227 
228     virtual void notifyConfigurationChanged(const NotifyConfigurationChangedArgs* args) override;
229     virtual void notifyKey(const NotifyKeyArgs* args) override;
230     virtual void notifyMotion(const NotifyMotionArgs* args) override;
231     virtual void notifySwitch(const NotifySwitchArgs* args) override;
232     virtual void notifyDeviceReset(const NotifyDeviceResetArgs* args) override;
233 
234     virtual void dump(std::string& dump) override;
235 
236     ~InputClassifier();
237 
238     // Called from InputManager
239     virtual void setMotionClassifierEnabled(bool enabled) override;
240 
241 private:
242     // Protect access to mMotionClassifier, since it may become null via a hidl callback
243     std::mutex mLock;
244     // The next stage to pass input events to
245     sp<InputListenerInterface> mListener;
246 
247     std::unique_ptr<MotionClassifierInterface> mMotionClassifier GUARDED_BY(mLock);
248     std::thread mInitializeMotionClassifierThread;
249     /**
250      * Set the value of mMotionClassifier.
251      * This is called from 2 different threads:
252      * 1) mInitializeMotionClassifierThread, when we have constructed a MotionClassifier
253      * 2) A binder thread of the HalDeathRecipient, which is created when HAL dies. This would cause
254      *    mMotionClassifier to become nullptr.
255      */
256     void setMotionClassifier(std::unique_ptr<MotionClassifierInterface> motionClassifier);
257 
258     /**
259      * The deathRecipient will call setMotionClassifier(null) when the HAL dies.
260      */
261     class HalDeathRecipient : public android::hardware::hidl_death_recipient {
262     public:
263         explicit HalDeathRecipient(InputClassifier& parent);
264         virtual void serviceDied(uint64_t cookie,
265                                  const wp<android::hidl::base::V1_0::IBase>& who) override;
266 
267     private:
268         InputClassifier& mParent;
269     };
270     // We retain a reference to death recipient, because the death recipient will be calling
271     // ~MotionClassifier if the HAL dies.
272     // If we don't retain a reference, and MotionClassifier is the only owner of the death
273     // recipient, the serviceDied call will cause death recipient to call its own destructor.
274     sp<HalDeathRecipient> mHalDeathRecipient;
275 };
276 
277 } // namespace android
278 #endif
279