1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server.wifi;
18 
19 import android.annotation.IntDef;
20 import android.content.Context;
21 import android.util.Log;
22 
23 import com.android.server.wifi.proto.WifiStatsLog;
24 import com.android.wifi.resources.R;
25 
26 import java.lang.annotation.Retention;
27 import java.lang.annotation.RetentionPolicy;
28 import java.util.Iterator;
29 import java.util.LinkedList;
30 import java.util.concurrent.TimeUnit;
31 
32 /**
33  * This class is used to recover the wifi stack from a fatal failure. The recovery mechanism
34  * involves triggering a stack restart (essentially simulating an airplane mode toggle) using
35  * {@link ActiveModeWarden}.
36  * The current triggers for:
37  * 1. Last resort watchdog bite.
38  * 2. HAL/wificond crashes during normal operation.
39  * 3. TBD: supplicant crashes during normal operation.
40  */
41 public class SelfRecovery {
42     private static final String TAG = "WifiSelfRecovery";
43 
44     /**
45      * Reason codes for the various recovery triggers.
46      */
47     public static final int REASON_LAST_RESORT_WATCHDOG = 0;
48     public static final int REASON_WIFINATIVE_FAILURE = 1;
49     public static final int REASON_STA_IFACE_DOWN = 2;
50     public static final int REASON_API_CALL = 3;
51     public static final int REASON_SUBSYSTEM_RESTART = 4;
52     public static final int REASON_IFACE_ADDED = 5;
53 
54     @Retention(RetentionPolicy.SOURCE)
55     @IntDef(prefix = {"REASON_"}, value = {
56             REASON_LAST_RESORT_WATCHDOG,
57             REASON_WIFINATIVE_FAILURE,
58             REASON_STA_IFACE_DOWN,
59             REASON_API_CALL,
60             REASON_SUBSYSTEM_RESTART,
61             REASON_IFACE_ADDED})
62     public @interface RecoveryReason {}
63 
64     /**
65      * State for self recovery.
66      */
67     private static final int STATE_NO_RECOVERY = 0;
68     private static final int STATE_DISABLE_WIFI = 1;
69     private static final int STATE_RESTART_WIFI = 2;
70 
71     @Retention(RetentionPolicy.SOURCE)
72     @IntDef(prefix = {"STATE_"}, value = {
73             STATE_NO_RECOVERY,
74             STATE_DISABLE_WIFI,
75             STATE_RESTART_WIFI})
76     private @interface RecoveryState {}
77 
78     private final Context mContext;
79     private final ActiveModeWarden mActiveModeWarden;
80     private final Clock mClock;
81     // Time since boot (in millis) that restart occurred
82     private final LinkedList<Long> mPastRestartTimes;
83     private final WifiNative mWifiNative;
84     private final WifiGlobals mWifiGlobals;
85     private int mSelfRecoveryReason;
86     private long mLastSelfRecoveryTimeStampMillis = -1L;
87     // Self recovery state
88     private @RecoveryState int mRecoveryState;
89     private SubsystemRestartListenerInternal mSubsystemRestartListener;
90 
91     /**
92      * Return the recovery reason code as string.
93      * @param reason the reason code
94      * @return the recovery reason as string
95      */
getRecoveryReasonAsString(@ecoveryReason int reason)96     public static String getRecoveryReasonAsString(@RecoveryReason int reason) {
97         switch (reason) {
98             case REASON_LAST_RESORT_WATCHDOG:
99                 return "Last Resort Watchdog";
100             case REASON_WIFINATIVE_FAILURE:
101                 return "WifiNative Failure";
102             case REASON_STA_IFACE_DOWN:
103                 return "Sta Interface Down";
104             case REASON_API_CALL:
105                 return "API call (e.g. user)";
106             case REASON_SUBSYSTEM_RESTART:
107                 return "Subsystem Restart";
108             case REASON_IFACE_ADDED:
109                 return "Interface Added";
110             default:
111                 return "Unknown " + reason;
112         }
113     }
114 
115     /**
116      * Invoked when self recovery completed.
117      */
onRecoveryCompleted()118     public void onRecoveryCompleted() {
119         mRecoveryState = STATE_NO_RECOVERY;
120     }
121 
122     /**
123      * Invoked when Wifi is stopped with all client mode managers removed.
124      */
onWifiStopped()125     public void onWifiStopped() {
126         if (mRecoveryState == STATE_DISABLE_WIFI) {
127             onRecoveryCompleted();
128         }
129     }
130 
131     /**
132      * Returns true if recovery is currently in progress.
133      */
isRecoveryInProgress()134     public boolean isRecoveryInProgress() {
135         // return true if in recovery progress
136         return mRecoveryState != STATE_NO_RECOVERY;
137     }
138 
139     private class SubsystemRestartListenerInternal
140             implements HalDeviceManager.SubsystemRestartListener{
onSubsystemRestart(@ecoveryReason int reason, long timeElapsedFromLastTrigger, int resultForLogging)141         private void onSubsystemRestart(@RecoveryReason int reason, long timeElapsedFromLastTrigger,
142                 int resultForLogging) {
143             Log.e(TAG, "Restarting wifi for reason: " + getRecoveryReasonAsString(reason));
144             WifiStatsLog.write(WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED,
145                     convertSelfRecoveryReason(reason),
146                     resultForLogging,
147                     timeElapsedFromLastTrigger);
148             mActiveModeWarden.recoveryRestartWifi(reason,
149                     reason != REASON_LAST_RESORT_WATCHDOG && reason != REASON_API_CALL);
150         }
151 
152         @Override
onSubsystemRestart()153         public void onSubsystemRestart() {
154             long timeElapsedFromLastTrigger = getTimeElapsedFromLastTrigger();
155             mLastSelfRecoveryTimeStampMillis = mClock.getWallClockMillis();
156             if (mRecoveryState == STATE_RESTART_WIFI) {
157                 // If the wifi restart recovery is triggered then proceed
158                 onSubsystemRestart(mSelfRecoveryReason, timeElapsedFromLastTrigger,
159                         WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_RESTART_STARTED_INTERNAL_RECOVERY_BY_NATIVE_CALLBACK);
160             } else {
161                 // We did not trigger recovery, but looks like the firmware crashed?
162                 mRecoveryState = STATE_RESTART_WIFI;
163                 onSubsystemRestart(REASON_SUBSYSTEM_RESTART, timeElapsedFromLastTrigger,
164                         WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_RESTART_STARTED_INTERNAL_RECOVERY_BY_NATIVE_CALLBACK);
165             }
166         }
167     }
168 
SelfRecovery(Context context, ActiveModeWarden activeModeWarden, Clock clock, WifiNative wifiNative, WifiGlobals wifiGlobals)169     public SelfRecovery(Context context, ActiveModeWarden activeModeWarden,
170             Clock clock, WifiNative wifiNative, WifiGlobals wifiGlobals) {
171         mContext = context;
172         mActiveModeWarden = activeModeWarden;
173         mClock = clock;
174         mPastRestartTimes = new LinkedList<>();
175         mWifiNative = wifiNative;
176         mSubsystemRestartListener = new SubsystemRestartListenerInternal();
177         mWifiNative.registerSubsystemRestartListener(mSubsystemRestartListener);
178         mWifiGlobals = wifiGlobals;
179         mRecoveryState = STATE_NO_RECOVERY;
180     }
181 
182     /**
183      * Trigger recovery.
184      *
185      * This method does the following:
186      * 1. Checks reason code used to trigger recovery
187      * 2. Checks for sta iface down triggers and disables wifi by sending {@link
188      * ActiveModeWarden#recoveryDisableWifi()} to {@link ActiveModeWarden} to disable wifi.
189      * 3. Throttles restart calls for underlying native failures
190      * 4. Sends {@link ActiveModeWarden#recoveryRestartWifi(int)} to {@link ActiveModeWarden} to
191      * initiate the stack restart.
192      * @param reason One of the above |REASON_*| codes.
193      */
trigger(@ecoveryReason int reason)194     public void trigger(@RecoveryReason int reason) {
195         long timeElapsedFromLastTrigger = getTimeElapsedFromLastTrigger();
196         mLastSelfRecoveryTimeStampMillis = mClock.getWallClockMillis();
197         if (!(reason == REASON_LAST_RESORT_WATCHDOG || reason == REASON_WIFINATIVE_FAILURE
198                   || reason == REASON_STA_IFACE_DOWN || reason == REASON_API_CALL
199                   || reason == REASON_IFACE_ADDED)) {
200             Log.e(TAG, "Invalid trigger reason. Ignoring...");
201             WifiStatsLog.write(WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED,
202                     convertSelfRecoveryReason(reason),
203                     WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_INVALID_REASON,
204                     timeElapsedFromLastTrigger);
205             return;
206         }
207         if (reason == REASON_STA_IFACE_DOWN) {
208             Log.e(TAG, "STA interface down, disable wifi");
209             mActiveModeWarden.recoveryDisableWifi();
210             mRecoveryState = STATE_DISABLE_WIFI;
211             WifiStatsLog.write(WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED,
212                     convertSelfRecoveryReason(reason),
213                     WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_IFACE_DOWN,
214                     timeElapsedFromLastTrigger);
215             return;
216         }
217 
218         Log.e(TAG, "Triggering recovery for reason: " + getRecoveryReasonAsString(reason));
219         if (reason == REASON_IFACE_ADDED
220                 && !mWifiGlobals.isWifiInterfaceAddedSelfRecoveryEnabled()) {
221             Log.w(TAG, "Recovery on added interface is disabled. Ignoring...");
222             WifiStatsLog.write(WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED,
223                     convertSelfRecoveryReason(reason),
224                     WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_IFACE_ADD_DISABLED,
225                     timeElapsedFromLastTrigger);
226             return;
227         }
228         if (reason == REASON_WIFINATIVE_FAILURE) {
229             int maxRecoveriesPerHour = mContext.getResources().getInteger(
230                     R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour);
231             if (maxRecoveriesPerHour == 0) {
232                 Log.e(TAG, "Recovery disabled. Disabling wifi");
233                 mActiveModeWarden.recoveryDisableWifi();
234                 mRecoveryState = STATE_DISABLE_WIFI;
235                 WifiStatsLog.write(WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED,
236                         convertSelfRecoveryReason(reason),
237                         WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_RETRY_DISABLED,
238                         timeElapsedFromLastTrigger);
239                 return;
240             }
241             trimPastRestartTimes();
242             if (mPastRestartTimes.size() >= maxRecoveriesPerHour) {
243                 Log.e(TAG, "Already restarted wifi " + maxRecoveriesPerHour + " times in"
244                         + " last 1 hour. Disabling wifi");
245                 mActiveModeWarden.recoveryDisableWifi();
246                 mRecoveryState = STATE_DISABLE_WIFI;
247                 WifiStatsLog.write(WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED,
248                         convertSelfRecoveryReason(reason),
249                         WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_ABOVE_MAX_RETRY,
250                         timeElapsedFromLastTrigger);
251                 return;
252             }
253             mPastRestartTimes.add(mClock.getElapsedSinceBootMillis());
254         }
255 
256         mSelfRecoveryReason = reason;
257         mRecoveryState = STATE_RESTART_WIFI;
258         if (!mWifiNative.startSubsystemRestart()) {
259             // HAL call failed, fallback to internal flow.
260             mSubsystemRestartListener.onSubsystemRestart(reason, timeElapsedFromLastTrigger,
261                     WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_RESTART_STARTED_INTERNAL_RECOVERY);
262             return;
263         }
264         WifiStatsLog.write(WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED,
265                 convertSelfRecoveryReason(reason),
266                 WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__RESULT__RES_RESTART_SUCCESS,
267                 timeElapsedFromLastTrigger);
268     }
269 
270     /**
271      * Process the mPastRestartTimes list, removing elements outside the max restarts time window
272      */
trimPastRestartTimes()273     private void trimPastRestartTimes() {
274         Iterator<Long> iter = mPastRestartTimes.iterator();
275         long now = mClock.getElapsedSinceBootMillis();
276         while (iter.hasNext()) {
277             Long restartTimeMillis = iter.next();
278             if (now - restartTimeMillis > TimeUnit.HOURS.toMillis(1)) {
279                 iter.remove();
280             } else {
281                 break;
282             }
283         }
284     }
285 
convertSelfRecoveryReason(int reason)286     private int convertSelfRecoveryReason(int reason) {
287         switch (reason) {
288             case SelfRecovery.REASON_LAST_RESORT_WATCHDOG:
289                 return WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__REASON__REASON_LAST_RESORT_WDOG;
290             case SelfRecovery.REASON_WIFINATIVE_FAILURE:
291                 return WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__REASON__REASON_WIFINATIVE_FAILURE;
292             case SelfRecovery.REASON_STA_IFACE_DOWN:
293                 return WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__REASON__REASON_STA_IFACE_DOWN;
294             case SelfRecovery.REASON_API_CALL:
295                 return WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__REASON__REASON_API_CALL;
296             case SelfRecovery.REASON_SUBSYSTEM_RESTART:
297                 return WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__REASON__REASON_SUBSYSTEM_RESTART;
298             case SelfRecovery.REASON_IFACE_ADDED:
299                 return WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__REASON__REASON_IFACE_ADDED;
300             default:
301                 return WifiStatsLog.WIFI_SELF_RECOVERY_TRIGGERED__REASON__REASON_UNKNOWN;
302         }
303     }
304 
getTimeElapsedFromLastTrigger()305     private long getTimeElapsedFromLastTrigger() {
306         if (mLastSelfRecoveryTimeStampMillis < 0) {
307             return -1L;
308         } else {
309             return (mClock.getWallClockMillis() - mLastSelfRecoveryTimeStampMillis);
310         }
311     }
312 }
313