1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef _UI_INPUT_CLASSIFIER_H
18 #define _UI_INPUT_CLASSIFIER_H
19 
20 #include <android-base/thread_annotations.h>
21 #include <utils/RefBase.h>
22 #include <unordered_map>
23 #include <thread>
24 
25 #include "BlockingQueue.h"
26 #include "InputListener.h"
27 #include <android/hardware/input/classifier/1.0/IInputClassifier.h>
28 
29 namespace android {
30 
31 enum class ClassifierEventType : uint8_t {
32     MOTION = 0,
33     DEVICE_RESET = 1,
34     HAL_RESET = 2,
35     EXIT = 3,
36 };
37 
38 struct ClassifierEvent {
39     ClassifierEventType type;
40     std::unique_ptr<NotifyArgs> args;
41 
42     ClassifierEvent(ClassifierEventType type, std::unique_ptr<NotifyArgs> args);
43     ClassifierEvent(std::unique_ptr<NotifyMotionArgs> args);
44     ClassifierEvent(std::unique_ptr<NotifyDeviceResetArgs> args);
45     ClassifierEvent(ClassifierEvent&& other);
46     ClassifierEvent& operator=(ClassifierEvent&& other);
47 
48     // Convenience function to create a HAL_RESET event
49     static ClassifierEvent createHalResetEvent();
50     // Convenience function to create an EXIT event
51     static ClassifierEvent createExitEvent();
52 
53     std::optional<int32_t> getDeviceId() const;
54 };
55 
56 // --- Interfaces ---
57 
58 /**
59  * Interface for adding a MotionClassification to NotifyMotionArgs.
60  *
61  * To implement, override the classify function.
62  */
63 class MotionClassifierInterface {
64 public:
MotionClassifierInterface()65     MotionClassifierInterface() { }
~MotionClassifierInterface()66     virtual ~MotionClassifierInterface() { }
67     /**
68      * Based on the motion event described by NotifyMotionArgs,
69      * provide a MotionClassification for the current gesture.
70      */
71     virtual MotionClassification classify(const NotifyMotionArgs& args) = 0;
72     /**
73      * Reset all internal HAL state.
74      */
75     virtual void reset() = 0;
76     /**
77      * Reset HAL state for a specific device.
78      */
79     virtual void reset(const NotifyDeviceResetArgs& args) = 0;
80 
81     /**
82      * Dump the state of the motion classifier
83      */
84     virtual void dump(std::string& dump) = 0;
85 };
86 
87 /**
88  * Base interface for an InputListener stage.
89  * Provides classification to events.
90  */
91 class InputClassifierInterface : public virtual RefBase, public InputListenerInterface {
92 public:
93     /**
94      * Dump the state of the input classifier.
95      * This method may be called on any thread (usually by the input manager).
96      */
97     virtual void dump(std::string& dump) = 0;
98 protected:
InputClassifierInterface()99     InputClassifierInterface() { }
~InputClassifierInterface()100     virtual ~InputClassifierInterface() { }
101 };
102 
103 // --- Implementations ---
104 
105 /**
106  * Implementation of MotionClassifierInterface that calls the InputClassifier HAL
107  * in order to determine the classification for the current gesture.
108  *
109  * The InputClassifier HAL may keep track of the entire gesture in order to determine
110  * the classification, and may be hardware-specific. It may use the data in
111  * NotifyMotionArgs::videoFrames field to drive the classification decisions.
112  * The HAL is called from a separate thread.
113  */
114 class MotionClassifier final : public MotionClassifierInterface {
115 public:
116     /**
117      * The deathRecipient will be subscribed to the HAL death. If the death recipient
118      * owns MotionClassifier and receives HAL death, it should delete its copy of it.
119      * The callback serviceDied will also be sent if the MotionClassifier itself fails
120      * to initialize. If the MotionClassifier fails to initialize, it is not useful, and
121      * should be deleted.
122      * If no death recipient is supplied, then the registration step will be skipped, so there will
123      * be no listeners registered for the HAL death. This is useful for testing
124      * MotionClassifier in isolation.
125      */
126     explicit MotionClassifier(sp<android::hardware::hidl_death_recipient> deathRecipient = nullptr);
127     ~MotionClassifier();
128 
129     /**
130      * Classifies events asynchronously; that is, it doesn't block events on a classification,
131      * but instead sends them over to the classifier HAL and after a classification is
132      * determined, it then marks the next event it sees in the stream with it.
133      *
134      * Therefore, it is acceptable to have the classifications be delayed by 1-2 events
135      * in a particular gesture.
136      */
137     virtual MotionClassification classify(const NotifyMotionArgs& args) override;
138     virtual void reset() override;
139     virtual void reset(const NotifyDeviceResetArgs& args) override;
140 
141     virtual void dump(std::string& dump) override;
142 
143 private:
144     /**
145      * Initialize MotionClassifier.
146      * Return true if initializaion is successful.
147      */
148     bool init();
149     /**
150      * Entity that will be notified of the HAL death (most likely InputClassifier).
151      */
152     wp<android::hardware::hidl_death_recipient> mDeathRecipient;
153 
154     // The events that need to be sent to the HAL.
155     BlockingQueue<ClassifierEvent> mEvents;
156     /**
157      * Add an event to the queue mEvents.
158      */
159     void enqueueEvent(ClassifierEvent&& event);
160     /**
161      * Thread that will communicate with InputClassifier HAL.
162      * This should be the only thread that communicates with InputClassifier HAL,
163      * because this thread is allowed to block on the HAL calls.
164      */
165     std::thread mHalThread;
166     /**
167      * Print an error message if the caller is not on the InputClassifier thread.
168      * Caller must supply the name of the calling function as __func__
169      */
170     void ensureHalThread(const char* function);
171     /**
172      * Call the InputClassifier HAL
173      */
174     void callInputClassifierHal();
175     /**
176      * Access to the InputClassifier HAL. May be null if init() hasn't completed yet.
177      * When init() successfully completes, mService is guaranteed to remain non-null and to not
178      * change its value until MotionClassifier is destroyed.
179      * This variable is *not* guarded by mLock in the InputClassifier thread, because
180      * that thread knows exactly when this variable is initialized.
181      * When accessed in any other thread, mService is checked for nullness with a lock.
182      */
183     sp<android::hardware::input::classifier::V1_0::IInputClassifier> mService;
184     std::mutex mLock;
185     /**
186      * Per-device input classifications. Should only be accessed using the
187      * getClassification / setClassification methods.
188      */
189     std::unordered_map<int32_t /*deviceId*/, MotionClassification>
190             mClassifications GUARDED_BY(mLock);
191     /**
192      * Set the current classification for a given device.
193      */
194     void setClassification(int32_t deviceId, MotionClassification classification);
195     /**
196      * Get the current classification for a given device.
197      */
198     MotionClassification getClassification(int32_t deviceId);
199     void updateClassification(int32_t deviceId, nsecs_t eventTime,
200             MotionClassification classification);
201     /**
202      * Clear all current classifications
203      */
204     void clearClassifications();
205     /**
206      * Per-device times when the last ACTION_DOWN was received.
207      * Used to reject late classifications that do not belong to the current gesture.
208      *
209      * Accessed indirectly by both InputClassifier thread and the thread that receives notifyMotion.
210      */
211     std::unordered_map<int32_t /*deviceId*/, nsecs_t /*downTime*/> mLastDownTimes GUARDED_BY(mLock);
212 
213     void updateLastDownTime(int32_t deviceId, nsecs_t downTime);
214 
215     /**
216      * Exit the InputClassifier HAL thread.
217      * Useful for tests to ensure proper cleanup.
218      */
219     void requestExit();
220     /**
221      * Return string status of mService
222      */
223     const char* getServiceStatus() REQUIRES(mLock);
224 };
225 
226 
227 /**
228  * Implementation of the InputClassifierInterface.
229  * Represents a separate stage of input processing. All of the input events go through this stage.
230  * Acts as a passthrough for all input events except for motion events.
231  * The events of motion type are sent to MotionClassifier.
232  */
233 class InputClassifier : public InputClassifierInterface,
234         public android::hardware::hidl_death_recipient {
235 public:
236     explicit InputClassifier(const sp<InputListenerInterface>& listener);
237     // Some of the constructor logic is finished in onFirstRef
238     virtual void onFirstRef() override;
239 
240     virtual void notifyConfigurationChanged(const NotifyConfigurationChangedArgs* args) override;
241     virtual void notifyKey(const NotifyKeyArgs* args) override;
242     virtual void notifyMotion(const NotifyMotionArgs* args) override;
243     virtual void notifySwitch(const NotifySwitchArgs* args) override;
244     virtual void notifyDeviceReset(const NotifyDeviceResetArgs* args) override;
245 
246     virtual void serviceDied(uint64_t cookie,
247             const wp<android::hidl::base::V1_0::IBase>& who) override;
248 
249     virtual void dump(std::string& dump) override;
250 
251 private:
252     // Protect access to mMotionClassifier, since it may become null via a hidl callback
253     std::mutex mLock;
254     std::unique_ptr<MotionClassifierInterface> mMotionClassifier GUARDED_BY(mLock);
255     // The next stage to pass input events to
256     sp<InputListenerInterface> mListener;
257 };
258 
259 } // namespace android
260 #endif
261