1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server;
18 
19 import static com.android.dx.mockito.inline.extended.ExtendedMockito.any;
20 import static com.android.dx.mockito.inline.extended.ExtendedMockito.anyBoolean;
21 import static com.android.dx.mockito.inline.extended.ExtendedMockito.anyInt;
22 import static com.android.dx.mockito.inline.extended.ExtendedMockito.anyLong;
23 import static com.android.dx.mockito.inline.extended.ExtendedMockito.anyString;
24 import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;
25 import static com.android.dx.mockito.inline.extended.ExtendedMockito.doReturn;
26 import static com.android.dx.mockito.inline.extended.ExtendedMockito.verify;
27 import static com.android.dx.mockito.inline.extended.ExtendedMockito.when;
28 import static com.android.server.RescueParty.DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN;
29 import static com.android.server.RescueParty.LEVEL_FACTORY_RESET;
30 import static com.android.server.RescueParty.RESCUE_LEVEL_FACTORY_RESET;
31 
32 import static org.junit.Assert.assertEquals;
33 import static org.junit.Assert.assertFalse;
34 import static org.junit.Assert.assertTrue;
35 import static org.mockito.ArgumentMatchers.eq;
36 import static org.mockito.ArgumentMatchers.isNull;
37 import static org.mockito.Mockito.never;
38 import static org.mockito.Mockito.spy;
39 import static org.mockito.Mockito.times;
40 
41 import android.content.ContentResolver;
42 import android.content.Context;
43 import android.content.pm.ApplicationInfo;
44 import android.content.pm.PackageManager;
45 import android.content.pm.VersionedPackage;
46 import android.crashrecovery.flags.Flags;
47 import android.os.RecoverySystem;
48 import android.os.SystemProperties;
49 import android.os.UserHandle;
50 import android.platform.test.flag.junit.SetFlagsRule;
51 import android.provider.DeviceConfig;
52 import android.provider.Settings;
53 import android.util.ArraySet;
54 
55 import com.android.dx.mockito.inline.extended.ExtendedMockito;
56 import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
57 import com.android.server.RescueParty.RescuePartyObserver;
58 import com.android.server.am.SettingsToPropertiesMapper;
59 
60 import org.junit.After;
61 import org.junit.Before;
62 import org.junit.Rule;
63 import org.junit.Test;
64 import org.mockito.Answers;
65 import org.mockito.ArgumentCaptor;
66 import org.mockito.Captor;
67 import org.mockito.Mock;
68 import org.mockito.MockitoSession;
69 import org.mockito.quality.Strictness;
70 import org.mockito.stubbing.Answer;
71 
72 import java.lang.reflect.Field;
73 import java.util.Arrays;
74 import java.util.HashMap;
75 import java.util.HashSet;
76 import java.util.List;
77 import java.util.Map;
78 import java.util.concurrent.Executor;
79 import java.util.concurrent.TimeUnit;
80 
81 /**
82  * Test RescueParty.
83  */
84 public class RescuePartyTest {
85     private static final long CURRENT_NETWORK_TIME_MILLIS = 0L;
86     private static final String FAKE_NATIVE_NAMESPACE1 = "native1";
87     private static final String FAKE_NATIVE_NAMESPACE2 = "native2";
88     private static final String[] FAKE_RESET_NATIVE_NAMESPACES =
89             {FAKE_NATIVE_NAMESPACE1, FAKE_NATIVE_NAMESPACE2};
90 
91     private static VersionedPackage sFailingPackage = new VersionedPackage("com.package.name", 1);
92     private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";
93     private static final String CALLING_PACKAGE1 = "com.package.name1";
94     private static final String CALLING_PACKAGE2 = "com.package.name2";
95     private static final String CALLING_PACKAGE3 = "com.package.name3";
96     private static final String PERSISTENT_PACKAGE = "com.persistent.package";
97     private static final String NON_PERSISTENT_PACKAGE = "com.nonpersistent.package";
98     private static final String NAMESPACE1 = "namespace1";
99     private static final String NAMESPACE2 = "namespace2";
100     private static final String NAMESPACE3 = "namespace3";
101     private static final String NAMESPACE4 = "namespace4";
102     private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
103             "persist.device_config.configuration.disable_rescue_party";
104     private static final String PROP_DISABLE_FACTORY_RESET_FLAG =
105             "persist.device_config.configuration.disable_rescue_party_factory_reset";
106 
107     @Rule
108     public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
109 
110     private MockitoSession mSession;
111     private HashMap<String, String> mSystemSettingsMap;
112     private HashMap<String, String> mCrashRecoveryPropertiesMap;
113     //Records the namespaces wiped by setProperties().
114     private HashSet<String> mNamespacesWiped;
115 
116     @Mock(answer = Answers.RETURNS_DEEP_STUBS)
117     private Context mMockContext;
118     @Mock(answer = Answers.RETURNS_DEEP_STUBS)
119     private PackageWatchdog mMockPackageWatchdog;
120     @Mock(answer = Answers.RETURNS_DEEP_STUBS)
121     private ContentResolver mMockContentResolver;
122     @Mock(answer = Answers.RETURNS_DEEP_STUBS)
123     private PackageManager mPackageManager;
124 
125     // Mock only sysprop apis
126     private PackageWatchdog.BootThreshold mSpyBootThreshold;
127 
128     @Captor
129     private ArgumentCaptor<DeviceConfig.MonitorCallback> mMonitorCallbackCaptor;
130     @Captor
131     private ArgumentCaptor<List<String>> mPackageListCaptor;
132 
133     @Before
setUp()134     public void setUp() throws Exception {
135         mSession =
136                 ExtendedMockito.mockitoSession().initMocks(
137                         this)
138                         .strictness(Strictness.LENIENT)
139                         .spyStatic(DeviceConfig.class)
140                         .spyStatic(SystemProperties.class)
141                         .spyStatic(Settings.Global.class)
142                         .spyStatic(Settings.Secure.class)
143                         .spyStatic(SettingsToPropertiesMapper.class)
144                         .spyStatic(RecoverySystem.class)
145                         .spyStatic(RescueParty.class)
146                         .spyStatic(PackageWatchdog.class)
147                         .startMocking();
148         mSystemSettingsMap = new HashMap<>();
149         mNamespacesWiped = new HashSet<>();
150 
151         when(mMockContext.getContentResolver()).thenReturn(mMockContentResolver);
152         when(mMockContext.getPackageManager()).thenReturn(mPackageManager);
153         ApplicationInfo persistentApplicationInfo = new ApplicationInfo();
154         persistentApplicationInfo.flags |=
155                 ApplicationInfo.FLAG_SYSTEM | ApplicationInfo.FLAG_PERSISTENT;
156 
157         // If the package name is PERSISTENT_PACKAGE, then set the flags to be persistent and
158         // system. Don't set any flags otherwise.
159         when(mPackageManager.getApplicationInfo(eq(PERSISTENT_PACKAGE),
160                 anyInt())).thenReturn(persistentApplicationInfo);
161         when(mPackageManager.getApplicationInfo(eq(NON_PERSISTENT_PACKAGE),
162                 anyInt())).thenReturn(new ApplicationInfo());
163         // Reset observer instance to get new mock context on every run
164         RescuePartyObserver.reset();
165 
166         // Mock SystemProperties setter and various getters
167         doAnswer((Answer<Void>) invocationOnMock -> {
168                     String key = invocationOnMock.getArgument(0);
169                     String value = invocationOnMock.getArgument(1);
170 
171                     mSystemSettingsMap.put(key, value);
172                     return null;
173                 }
174         ).when(() -> SystemProperties.set(anyString(), anyString()));
175 
176         doAnswer((Answer<Boolean>) invocationOnMock -> {
177                     String key = invocationOnMock.getArgument(0);
178                     boolean defaultValue = invocationOnMock.getArgument(1);
179 
180                     String storedValue = mSystemSettingsMap.get(key);
181                     return storedValue == null ? defaultValue : Boolean.parseBoolean(storedValue);
182                 }
183         ).when(() -> SystemProperties.getBoolean(anyString(), anyBoolean()));
184 
185         doAnswer((Answer<Integer>) invocationOnMock -> {
186                     String key = invocationOnMock.getArgument(0);
187                     int defaultValue = invocationOnMock.getArgument(1);
188 
189                     String storedValue = mSystemSettingsMap.get(key);
190                     return storedValue == null ? defaultValue : Integer.parseInt(storedValue);
191                 }
192         ).when(() -> SystemProperties.getInt(anyString(), anyInt()));
193 
194         doAnswer((Answer<Long>) invocationOnMock -> {
195                     String key = invocationOnMock.getArgument(0);
196                     long defaultValue = invocationOnMock.getArgument(1);
197 
198                     String storedValue = mSystemSettingsMap.get(key);
199                     return storedValue == null ? defaultValue : Long.parseLong(storedValue);
200                 }
201         ).when(() -> SystemProperties.getLong(anyString(), anyLong()));
202 
203         // Mock DeviceConfig
204         doAnswer((Answer<Boolean>) invocationOnMock -> true)
205                 .when(() -> DeviceConfig.setProperty(anyString(), anyString(), anyString(),
206                         anyBoolean()));
207         doAnswer((Answer<Void>) invocationOnMock -> null)
208                 .when(() -> DeviceConfig.resetToDefaults(anyInt(), anyString()));
209         doAnswer((Answer<Boolean>) invocationOnMock -> {
210                     DeviceConfig.Properties properties = invocationOnMock.getArgument(0);
211                     String namespace = properties.getNamespace();
212                     // record a wipe
213                     if (properties.getKeyset().isEmpty()) {
214                         mNamespacesWiped.add(namespace);
215                     }
216                     return true;
217                 }
218         ).when(() -> DeviceConfig.setProperties(any(DeviceConfig.Properties.class)));
219 
220         // Mock PackageWatchdog
221         doAnswer((Answer<PackageWatchdog>) invocationOnMock -> mMockPackageWatchdog)
222                 .when(() -> PackageWatchdog.getInstance(mMockContext));
223         mockCrashRecoveryProperties(mMockPackageWatchdog);
224 
225         doReturn(CURRENT_NETWORK_TIME_MILLIS).when(() -> RescueParty.getElapsedRealtime());
226 
227         setCrashRecoveryPropRescueBootCount(0);
228         SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
229         SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(false));
230 
231         // enable flag resets for tests
232         mSetFlagsRule.enableFlags(Flags.FLAG_ALLOW_RESCUE_PARTY_FLAG_RESETS);
233     }
234 
235     @After
tearDown()236     public void tearDown() throws Exception {
237         mSession.finishMocking();
238     }
239 
240     @Test
testBootLoopDetectionWithExecutionForAllRescueLevels()241     public void testBootLoopDetectionWithExecutionForAllRescueLevels() {
242         // this is old test where the flag needs to be disabled
243         mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
244 
245         RescueParty.onSettingsProviderPublished(mMockContext);
246         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
247                 any(Executor.class),
248                 mMonitorCallbackCaptor.capture()));
249         HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
250 
251         noteBoot(1);
252 
253         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, /*resetNamespaces=*/ null,
254                 verifiedTimesMap);
255 
256         // Record DeviceConfig accesses
257         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
258         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
259         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
260         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
261 
262         final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
263 
264         noteBoot(2);
265 
266         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, expectedAllResetNamespaces,
267                 verifiedTimesMap);
268 
269         noteBoot(3);
270 
271         verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, expectedAllResetNamespaces,
272                 verifiedTimesMap);
273 
274         noteBoot(4);
275         assertTrue(RescueParty.isRebootPropertySet());
276 
277         setCrashRecoveryPropAttemptingReboot(false);
278         noteBoot(5);
279         assertTrue(RescueParty.isFactoryResetPropertySet());
280     }
281 
282     @Test
testBootLoopDetectionWithExecutionForAllRescueLevelsRecoverabilityDetection()283     public void testBootLoopDetectionWithExecutionForAllRescueLevelsRecoverabilityDetection() {
284         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
285         RescueParty.onSettingsProviderPublished(mMockContext);
286         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
287                 any(Executor.class),
288                 mMonitorCallbackCaptor.capture()));
289         HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
290 
291         // Record DeviceConfig accesses
292         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
293         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
294         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
295 
296         final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
297 
298         noteBoot(1);
299         verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
300 
301         noteBoot(2);
302         assertTrue(RescueParty.isRebootPropertySet());
303 
304         noteBoot(3);
305         verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
306 
307         noteBoot(4);
308         verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
309 
310         noteBoot(5);
311         verifyOnlySettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
312 
313         setCrashRecoveryPropAttemptingReboot(false);
314         noteBoot(6);
315         assertTrue(RescueParty.isFactoryResetPropertySet());
316     }
317 
318     @Test
testPersistentAppCrashDetectionWithExecutionForAllRescueLevels()319     public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() {
320         // this is old test where the flag needs to be disabled
321         mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
322 
323         noteAppCrash(1, true);
324 
325         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, /*resetNamespaces=*/ null,
326                 /*configResetVerifiedTimesMap=*/ null);
327 
328         noteAppCrash(2, true);
329 
330         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, /*resetNamespaces=*/ null,
331                 /*configResetVerifiedTimesMap=*/ null);
332 
333         noteAppCrash(3, true);
334 
335         verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, /*resetNamespaces=*/ null,
336                 /*configResetVerifiedTimesMap=*/ null);
337 
338         noteAppCrash(4, true);
339         assertTrue(RescueParty.isRebootPropertySet());
340 
341         setCrashRecoveryPropAttemptingReboot(false);
342         noteAppCrash(5, true);
343         assertTrue(RescueParty.isFactoryResetPropertySet());
344     }
345 
346     @Test
testPersistentAppCrashDetectionWithExecutionForAllRescueLevelsRecoverability()347     public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevelsRecoverability() {
348         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
349         RescueParty.onSettingsProviderPublished(mMockContext);
350         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
351                 any(Executor.class),
352                 mMonitorCallbackCaptor.capture()));
353         HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
354 
355         // Record DeviceConfig accesses
356         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
357         monitorCallback.onDeviceConfigAccess(PERSISTENT_PACKAGE, NAMESPACE1);
358         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
359         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
360 
361         final String[] expectedResetNamespaces = new String[]{NAMESPACE1};
362         final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
363 
364         noteAppCrash(1, true);
365         verifyDeviceConfigReset(expectedResetNamespaces, verifiedTimesMap);
366 
367         noteAppCrash(2, true);
368         verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
369 
370         noteAppCrash(3, true);
371         assertTrue(RescueParty.isRebootPropertySet());
372 
373         noteAppCrash(4, true);
374         verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
375 
376         noteAppCrash(5, true);
377         verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
378 
379         noteAppCrash(6, true);
380         verifyOnlySettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
381 
382         setCrashRecoveryPropAttemptingReboot(false);
383         noteAppCrash(7, true);
384         assertTrue(RescueParty.isFactoryResetPropertySet());
385     }
386 
387     @Test
testNonPersistentAppOnlyPerformsFlagResets()388     public void testNonPersistentAppOnlyPerformsFlagResets() {
389         // this is old test where the flag needs to be disabled
390         mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
391 
392         noteAppCrash(1, false);
393 
394         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, /*resetNamespaces=*/ null,
395                 /*configResetVerifiedTimesMap=*/ null);
396 
397         noteAppCrash(2, false);
398 
399         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, /*resetNamespaces=*/ null,
400                 /*configResetVerifiedTimesMap=*/ null);
401 
402         noteAppCrash(3, false);
403 
404         verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, /*resetNamespaces=*/ null,
405                 /*configResetVerifiedTimesMap=*/ null);
406 
407         noteAppCrash(4, false);
408         assertFalse(RescueParty.isRebootPropertySet());
409 
410         noteAppCrash(5, false);
411         assertFalse(RescueParty.isFactoryResetPropertySet());
412     }
413 
414     @Test
testNonPersistentAppOnlyPerformsFlagResetsRecoverabilityDetection()415     public void testNonPersistentAppOnlyPerformsFlagResetsRecoverabilityDetection() {
416         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
417         RescueParty.onSettingsProviderPublished(mMockContext);
418         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
419                 any(Executor.class),
420                 mMonitorCallbackCaptor.capture()));
421         HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
422 
423         // Record DeviceConfig accesses
424         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
425         monitorCallback.onDeviceConfigAccess(NON_PERSISTENT_PACKAGE, NAMESPACE1);
426         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
427         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
428 
429         final String[] expectedResetNamespaces = new String[]{NAMESPACE1};
430         final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
431 
432         noteAppCrash(1, false);
433         verifyDeviceConfigReset(expectedResetNamespaces, verifiedTimesMap);
434 
435         noteAppCrash(2, false);
436         verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
437 
438         noteAppCrash(3, false);
439         assertFalse(RescueParty.isRebootPropertySet());
440 
441         noteAppCrash(4, false);
442         verifyNoSettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
443         noteAppCrash(5, false);
444         verifyNoSettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
445         noteAppCrash(6, false);
446         verifyNoSettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
447 
448         setCrashRecoveryPropAttemptingReboot(false);
449         noteAppCrash(7, false);
450         assertFalse(RescueParty.isFactoryResetPropertySet());
451     }
452 
453     @Test
testNonPersistentAppCrashDetectionWithScopedResets()454     public void testNonPersistentAppCrashDetectionWithScopedResets() {
455         // this is old test where the flag needs to be disabled
456         mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
457 
458         RescueParty.onSettingsProviderPublished(mMockContext);
459         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
460                 any(Executor.class),
461                 mMonitorCallbackCaptor.capture()));
462 
463         // Record DeviceConfig accesses
464         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
465         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
466         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
467         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
468         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE2);
469         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE3);
470 
471         // Fake DeviceConfig value changes
472         monitorCallback.onNamespaceUpdate(NAMESPACE1);
473         verify(mMockPackageWatchdog).startObservingHealth(observer,
474                 Arrays.asList(CALLING_PACKAGE1), RescueParty.DEFAULT_OBSERVING_DURATION_MS);
475         monitorCallback.onNamespaceUpdate(NAMESPACE2);
476         verify(mMockPackageWatchdog, times(2)).startObservingHealth(eq(observer),
477                 mPackageListCaptor.capture(),
478                 eq(RescueParty.DEFAULT_OBSERVING_DURATION_MS));
479         monitorCallback.onNamespaceUpdate(NAMESPACE3);
480         verify(mMockPackageWatchdog).startObservingHealth(observer,
481                 Arrays.asList(CALLING_PACKAGE2), RescueParty.DEFAULT_OBSERVING_DURATION_MS);
482         assertTrue(mPackageListCaptor.getValue().containsAll(
483                 Arrays.asList(CALLING_PACKAGE1, CALLING_PACKAGE2)));
484         // Perform and verify scoped resets
485         final String[] expectedResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
486         final String[] expectedAllResetNamespaces =
487                 new String[]{NAMESPACE1, NAMESPACE2, NAMESPACE3};
488         HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
489         observer.execute(new VersionedPackage(
490                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, 1);
491         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, expectedResetNamespaces,
492                 verifiedTimesMap);
493 
494         observer.execute(new VersionedPackage(
495                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2);
496         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, expectedResetNamespaces,
497                 verifiedTimesMap);
498 
499         observer.execute(new VersionedPackage(
500                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3);
501         verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, expectedAllResetNamespaces,
502                 verifiedTimesMap);
503 
504         observer.execute(new VersionedPackage(
505                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4);
506         assertFalse(RescueParty.isRebootPropertySet());
507 
508         observer.execute(new VersionedPackage(
509                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 5);
510         assertFalse(RescueParty.isFactoryResetPropertySet());
511     }
512 
513     @Test
testNonDeviceConfigSettingsOnlyResetOncePerLevel()514     public void testNonDeviceConfigSettingsOnlyResetOncePerLevel() {
515         // this is old test where the flag needs to be disabled
516         mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
517 
518         RescueParty.onSettingsProviderPublished(mMockContext);
519         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
520                 any(Executor.class),
521                 mMonitorCallbackCaptor.capture()));
522 
523         // Record DeviceConfig accesses
524         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
525         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
526         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
527         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
528         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE2);
529         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE3);
530         // Fake DeviceConfig value changes
531         monitorCallback.onNamespaceUpdate(NAMESPACE1);
532         monitorCallback.onNamespaceUpdate(NAMESPACE2);
533         monitorCallback.onNamespaceUpdate(NAMESPACE3);
534         // Perform and verify scoped resets
535         final String[] expectedPackage1ResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
536         final String[] expectedPackage2ResetNamespaces = new String[]{NAMESPACE2, NAMESPACE3};
537         final String[] expectedAllResetNamespaces =
538                 new String[]{NAMESPACE1, NAMESPACE2, NAMESPACE3};
539         HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
540         observer.execute(new VersionedPackage(
541                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, 1);
542         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS,
543                 expectedPackage1ResetNamespaces, verifiedTimesMap);
544 
545         // Settings.Global & Settings.Secure should still remain the same execution times.
546         observer.execute(new VersionedPackage(
547                 CALLING_PACKAGE2, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, 1);
548         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS,
549                 expectedPackage2ResetNamespaces, verifiedTimesMap);
550 
551         observer.execute(new VersionedPackage(
552                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2);
553         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES,
554                 expectedPackage1ResetNamespaces, verifiedTimesMap);
555 
556         // Settings.Global & Settings.Secure should still remain the same execution times.
557         observer.execute(new VersionedPackage(
558                 CALLING_PACKAGE2, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2);
559         verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES,
560                 expectedPackage2ResetNamespaces, verifiedTimesMap);
561 
562         observer.execute(new VersionedPackage(
563                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3);
564         verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, expectedAllResetNamespaces,
565                 verifiedTimesMap);
566 
567         // Settings.Global & Settings.Secure should still remain the same execution times.
568         observer.execute(new VersionedPackage(
569                 CALLING_PACKAGE2, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3);
570         verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, expectedAllResetNamespaces,
571                 verifiedTimesMap);
572 
573         observer.execute(new VersionedPackage(
574                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4);
575         assertFalse(RescueParty.isRebootPropertySet());
576 
577         observer.execute(new VersionedPackage(
578                 CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 5);
579         assertFalse(RescueParty.isFactoryResetPropertySet());
580     }
581 
582     @Test
testIsRecoveryTriggeredReboot()583     public void testIsRecoveryTriggeredReboot() {
584         for (int i = 0; i < LEVEL_FACTORY_RESET; i++) {
585             noteBoot(i + 1);
586         }
587         assertFalse(RescueParty.isFactoryResetPropertySet());
588         setCrashRecoveryPropAttemptingReboot(false);
589         noteBoot(LEVEL_FACTORY_RESET + 1);
590         assertTrue(RescueParty.isRecoveryTriggeredReboot());
591         assertTrue(RescueParty.isFactoryResetPropertySet());
592     }
593 
594     @Test
testIsRecoveryTriggeredRebootRecoverabilityDetection()595     public void testIsRecoveryTriggeredRebootRecoverabilityDetection() {
596         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
597         for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
598             noteBoot(i + 1);
599         }
600         assertFalse(RescueParty.isFactoryResetPropertySet());
601         setCrashRecoveryPropAttemptingReboot(false);
602         noteBoot(RESCUE_LEVEL_FACTORY_RESET + 1);
603         assertTrue(RescueParty.isRecoveryTriggeredReboot());
604         assertTrue(RescueParty.isFactoryResetPropertySet());
605     }
606 
607     @Test
testIsRecoveryTriggeredRebootOnlyAfterRebootCompleted()608     public void testIsRecoveryTriggeredRebootOnlyAfterRebootCompleted() {
609         for (int i = 0; i < LEVEL_FACTORY_RESET; i++) {
610             noteBoot(i + 1);
611         }
612         int mitigationCount = LEVEL_FACTORY_RESET + 1;
613         assertFalse(RescueParty.isFactoryResetPropertySet());
614         noteBoot(mitigationCount++);
615         assertFalse(RescueParty.isFactoryResetPropertySet());
616         noteBoot(mitigationCount++);
617         assertFalse(RescueParty.isFactoryResetPropertySet());
618         noteBoot(mitigationCount++);
619         setCrashRecoveryPropAttemptingReboot(false);
620         noteBoot(mitigationCount + 1);
621         assertTrue(RescueParty.isRecoveryTriggeredReboot());
622         assertTrue(RescueParty.isFactoryResetPropertySet());
623     }
624 
625     @Test
testIsRecoveryTriggeredRebootOnlyAfterRebootCompletedRecoverabilityDetection()626     public void testIsRecoveryTriggeredRebootOnlyAfterRebootCompletedRecoverabilityDetection() {
627         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
628         for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
629             noteBoot(i + 1);
630         }
631         int mitigationCount = RESCUE_LEVEL_FACTORY_RESET + 1;
632         assertFalse(RescueParty.isFactoryResetPropertySet());
633         noteBoot(mitigationCount++);
634         assertFalse(RescueParty.isFactoryResetPropertySet());
635         noteBoot(mitigationCount++);
636         assertFalse(RescueParty.isFactoryResetPropertySet());
637         noteBoot(mitigationCount++);
638         setCrashRecoveryPropAttemptingReboot(false);
639         noteBoot(mitigationCount + 1);
640         assertTrue(RescueParty.isRecoveryTriggeredReboot());
641         assertTrue(RescueParty.isFactoryResetPropertySet());
642     }
643 
644     @Test
testThrottlingOnBootFailures()645     public void testThrottlingOnBootFailures() {
646         setCrashRecoveryPropAttemptingReboot(false);
647         long now = System.currentTimeMillis();
648         long beforeTimeout = now - TimeUnit.MINUTES.toMillis(
649                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN - 1);
650         setCrashRecoveryPropLastFactoryReset(beforeTimeout);
651         for (int i = 1; i <= LEVEL_FACTORY_RESET; i++) {
652             noteBoot(i);
653         }
654         assertFalse(RescueParty.isRecoveryTriggeredReboot());
655     }
656 
657     @Test
testThrottlingOnBootFailuresRecoverabilityDetection()658     public void testThrottlingOnBootFailuresRecoverabilityDetection() {
659         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
660         setCrashRecoveryPropAttemptingReboot(false);
661         long now = System.currentTimeMillis();
662         long beforeTimeout = now - TimeUnit.MINUTES.toMillis(
663                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN - 1);
664         setCrashRecoveryPropLastFactoryReset(beforeTimeout);
665         for (int i = 1; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
666             noteBoot(i);
667         }
668         assertFalse(RescueParty.isRecoveryTriggeredReboot());
669     }
670 
671     @Test
testThrottlingOnAppCrash()672     public void testThrottlingOnAppCrash() {
673         setCrashRecoveryPropAttemptingReboot(false);
674         long now = System.currentTimeMillis();
675         long beforeTimeout = now - TimeUnit.MINUTES.toMillis(
676                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN - 1);
677         setCrashRecoveryPropLastFactoryReset(beforeTimeout);
678         for (int i = 0; i <= LEVEL_FACTORY_RESET; i++) {
679             noteAppCrash(i + 1, true);
680         }
681         assertFalse(RescueParty.isRecoveryTriggeredReboot());
682     }
683 
684     @Test
testThrottlingOnAppCrashRecoverabilityDetection()685     public void testThrottlingOnAppCrashRecoverabilityDetection() {
686         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
687         setCrashRecoveryPropAttemptingReboot(false);
688         long now = System.currentTimeMillis();
689         long beforeTimeout = now - TimeUnit.MINUTES.toMillis(
690                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN - 1);
691         setCrashRecoveryPropLastFactoryReset(beforeTimeout);
692         for (int i = 0; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
693             noteAppCrash(i + 1, true);
694         }
695         assertFalse(RescueParty.isRecoveryTriggeredReboot());
696     }
697 
698     @Test
testNotThrottlingAfterTimeoutOnBootFailures()699     public void testNotThrottlingAfterTimeoutOnBootFailures() {
700         setCrashRecoveryPropAttemptingReboot(false);
701         long now = System.currentTimeMillis();
702         long afterTimeout = now - TimeUnit.MINUTES.toMillis(
703                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN + 1);
704         setCrashRecoveryPropLastFactoryReset(afterTimeout);
705         for (int i = 1; i <= LEVEL_FACTORY_RESET; i++) {
706             noteBoot(i);
707         }
708         assertTrue(RescueParty.isRecoveryTriggeredReboot());
709     }
710 
711     @Test
testNotThrottlingAfterTimeoutOnBootFailuresRecoverabilityDetection()712     public void testNotThrottlingAfterTimeoutOnBootFailuresRecoverabilityDetection() {
713         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
714         setCrashRecoveryPropAttemptingReboot(false);
715         long now = System.currentTimeMillis();
716         long afterTimeout = now - TimeUnit.MINUTES.toMillis(
717                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN + 1);
718         setCrashRecoveryPropLastFactoryReset(afterTimeout);
719         for (int i = 1; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
720             noteBoot(i);
721         }
722         assertTrue(RescueParty.isRecoveryTriggeredReboot());
723     }
724 
725     @Test
testNotThrottlingAfterTimeoutOnAppCrash()726     public void testNotThrottlingAfterTimeoutOnAppCrash() {
727         setCrashRecoveryPropAttemptingReboot(false);
728         long now = System.currentTimeMillis();
729         long afterTimeout = now - TimeUnit.MINUTES.toMillis(
730                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN + 1);
731         setCrashRecoveryPropLastFactoryReset(afterTimeout);
732         for (int i = 0; i <= LEVEL_FACTORY_RESET; i++) {
733             noteAppCrash(i + 1, true);
734         }
735         assertTrue(RescueParty.isRecoveryTriggeredReboot());
736     }
737 
738     @Test
testNotThrottlingAfterTimeoutOnAppCrashRecoverabilityDetection()739     public void testNotThrottlingAfterTimeoutOnAppCrashRecoverabilityDetection() {
740         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
741         setCrashRecoveryPropAttemptingReboot(false);
742         long now = System.currentTimeMillis();
743         long afterTimeout = now - TimeUnit.MINUTES.toMillis(
744                 DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN + 1);
745         setCrashRecoveryPropLastFactoryReset(afterTimeout);
746         for (int i = 0; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
747             noteAppCrash(i + 1, true);
748         }
749         assertTrue(RescueParty.isRecoveryTriggeredReboot());
750     }
751 
752     @Test
testNativeRescuePartyResets()753     public void testNativeRescuePartyResets() {
754         doReturn(true).when(() -> SettingsToPropertiesMapper.isNativeFlagsResetPerformed());
755         doReturn(FAKE_RESET_NATIVE_NAMESPACES).when(
756                 () -> SettingsToPropertiesMapper.getResetNativeCategories());
757 
758         RescueParty.onSettingsProviderPublished(mMockContext);
759 
760         verify(() -> DeviceConfig.resetToDefaults(Settings.RESET_MODE_TRUSTED_DEFAULTS,
761                 FAKE_NATIVE_NAMESPACE1));
762         verify(() -> DeviceConfig.resetToDefaults(Settings.RESET_MODE_TRUSTED_DEFAULTS,
763                 FAKE_NATIVE_NAMESPACE2));
764     }
765 
766     @Test
testExplicitlyEnablingAndDisablingRescue()767     public void testExplicitlyEnablingAndDisablingRescue() {
768         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
769         SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
770         SystemProperties.set(PROP_DISABLE_RESCUE, Boolean.toString(true));
771         assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
772                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1), false);
773 
774         SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
775         assertTrue(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
776                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1));
777     }
778 
779     @Test
testDisablingRescueByDeviceConfigFlag()780     public void testDisablingRescueByDeviceConfigFlag() {
781         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
782         SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
783         SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(true));
784 
785         assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
786                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1), false);
787 
788         // Restore the property value initialized in SetUp()
789         SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
790         SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(false));
791     }
792 
793     @Test
testDisablingFactoryResetByDeviceConfigFlag()794     public void testDisablingFactoryResetByDeviceConfigFlag() {
795         SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, Boolean.toString(true));
796 
797         for (int i = 0; i < LEVEL_FACTORY_RESET; i++) {
798             noteBoot(i + 1);
799         }
800         assertFalse(RescueParty.isFactoryResetPropertySet());
801 
802         // Restore the property value initialized in SetUp()
803         SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, "");
804     }
805 
806     @Test
testDisablingFactoryResetByDeviceConfigFlagRecoverabilityDetection()807     public void testDisablingFactoryResetByDeviceConfigFlagRecoverabilityDetection() {
808         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
809         SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, Boolean.toString(true));
810 
811         for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
812             noteBoot(i + 1);
813         }
814         assertFalse(RescueParty.isFactoryResetPropertySet());
815 
816         // Restore the property value initialized in SetUp()
817         SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, "");
818     }
819 
820     @Test
testHealthCheckLevels()821     public void testHealthCheckLevels() {
822         // this is old test where the flag needs to be disabled
823         mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
824 
825         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
826 
827         // Ensure that no action is taken for cases where the failure reason is unknown
828         assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN, 1),
829                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
830 
831         // Ensure the correct user impact is returned for each mitigation count.
832         assertEquals(observer.onHealthCheckFailed(null,
833                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1),
834                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
835 
836         assertEquals(observer.onHealthCheckFailed(null,
837                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2),
838                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
839 
840         assertEquals(observer.onHealthCheckFailed(null,
841                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3),
842                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
843 
844         assertEquals(observer.onHealthCheckFailed(null,
845                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4),
846                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
847     }
848 
849     @Test
testHealthCheckLevelsRecoverabilityDetection()850     public void testHealthCheckLevelsRecoverabilityDetection() {
851         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
852         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
853 
854         // Ensure that no action is taken for cases where the failure reason is unknown
855         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
856                 PackageWatchdog.FAILURE_REASON_UNKNOWN, 1),
857                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
858 
859         // Ensure the correct user impact is returned for each mitigation count.
860         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
861                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1),
862                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
863 
864         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
865                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2),
866                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
867 
868         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
869                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3),
870                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
871 
872         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
873                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4),
874                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
875 
876         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
877                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 5),
878                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
879 
880         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
881                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 6),
882                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
883 
884         assertEquals(observer.onHealthCheckFailed(sFailingPackage,
885                 PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 7),
886                 PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
887     }
888 
889     @Test
testBootLoopLevels()890     public void testBootLoopLevels() {
891         // this is old test where the flag needs to be disabled
892         mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
893 
894         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
895 
896         assertEquals(observer.onBootLoop(0), PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
897         assertEquals(observer.onBootLoop(1), PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
898         assertEquals(observer.onBootLoop(2), PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
899         assertEquals(observer.onBootLoop(3), PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
900         assertEquals(observer.onBootLoop(4), PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
901         assertEquals(observer.onBootLoop(5), PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
902     }
903 
904     @Test
testBootLoopLevelsRecoverabilityDetection()905     public void testBootLoopLevelsRecoverabilityDetection() {
906         mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
907         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
908 
909         assertEquals(observer.onBootLoop(1), PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
910         assertEquals(observer.onBootLoop(2), PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
911         assertEquals(observer.onBootLoop(3), PackageHealthObserverImpact.USER_IMPACT_LEVEL_71);
912         assertEquals(observer.onBootLoop(4), PackageHealthObserverImpact.USER_IMPACT_LEVEL_75);
913         assertEquals(observer.onBootLoop(5), PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
914         assertEquals(observer.onBootLoop(6), PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
915     }
916 
917     @Test
testResetDeviceConfigForPackagesOnlyRuntimeMap()918     public void testResetDeviceConfigForPackagesOnlyRuntimeMap() {
919         RescueParty.onSettingsProviderPublished(mMockContext);
920         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
921                 any(Executor.class),
922                 mMonitorCallbackCaptor.capture()));
923 
924         // Record DeviceConfig accesses
925         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
926         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
927         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
928         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
929         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE2);
930         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE3);
931         // Fake DeviceConfig value changes
932         monitorCallback.onNamespaceUpdate(NAMESPACE1);
933         monitorCallback.onNamespaceUpdate(NAMESPACE2);
934         monitorCallback.onNamespaceUpdate(NAMESPACE3);
935 
936         doReturn("").when(() -> DeviceConfig.getString(
937                 eq(RescueParty.NAMESPACE_CONFIGURATION),
938                 eq(RescueParty.NAMESPACE_TO_PACKAGE_MAPPING_FLAG),
939                 eq("")));
940 
941         RescueParty.resetDeviceConfigForPackages(Arrays.asList(new String[]{CALLING_PACKAGE1}));
942         ArraySet<String> expectedNamespacesWiped = new ArraySet<String>(
943                 Arrays.asList(new String[]{NAMESPACE1, NAMESPACE2}));
944         assertEquals(mNamespacesWiped, expectedNamespacesWiped);
945     }
946 
947     @Test
testResetDeviceConfigForPackagesOnlyPresetMap()948     public void testResetDeviceConfigForPackagesOnlyPresetMap() {
949         RescueParty.onSettingsProviderPublished(mMockContext);
950         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
951                 any(Executor.class),
952                 mMonitorCallbackCaptor.capture()));
953 
954         String presetMapping = NAMESPACE1 + ":" + CALLING_PACKAGE1 + ","
955                 + NAMESPACE2 + ":" + CALLING_PACKAGE2 + ","
956                 + NAMESPACE3 +  ":" + CALLING_PACKAGE1;
957         doReturn(presetMapping).when(() -> DeviceConfig.getString(
958                 eq(RescueParty.NAMESPACE_CONFIGURATION),
959                 eq(RescueParty.NAMESPACE_TO_PACKAGE_MAPPING_FLAG),
960                 eq("")));
961 
962         RescueParty.resetDeviceConfigForPackages(Arrays.asList(new String[]{CALLING_PACKAGE1}));
963         ArraySet<String> expectedNamespacesWiped = new ArraySet<String>(
964                 Arrays.asList(new String[]{NAMESPACE1, NAMESPACE3}));
965         assertEquals(mNamespacesWiped, expectedNamespacesWiped);
966     }
967 
968     @Test
testResetDeviceConfigForPackagesBothMaps()969     public void testResetDeviceConfigForPackagesBothMaps() {
970         RescueParty.onSettingsProviderPublished(mMockContext);
971         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
972                 any(Executor.class),
973                 mMonitorCallbackCaptor.capture()));
974 
975         // Record DeviceConfig accesses
976         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
977         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
978         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
979         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
980         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE2);
981         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE3);
982         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE3, NAMESPACE4);
983         // Fake DeviceConfig value changes
984         monitorCallback.onNamespaceUpdate(NAMESPACE1);
985         monitorCallback.onNamespaceUpdate(NAMESPACE2);
986         monitorCallback.onNamespaceUpdate(NAMESPACE3);
987         monitorCallback.onNamespaceUpdate(NAMESPACE4);
988 
989         String presetMapping = NAMESPACE1 + ":" + CALLING_PACKAGE1 + ","
990                 + NAMESPACE2 + ":" + CALLING_PACKAGE2 + ","
991                 + NAMESPACE4 + ":" + CALLING_PACKAGE3;
992         doReturn(presetMapping).when(() -> DeviceConfig.getString(
993                 eq(RescueParty.NAMESPACE_CONFIGURATION),
994                 eq(RescueParty.NAMESPACE_TO_PACKAGE_MAPPING_FLAG),
995                 eq("")));
996 
997         RescueParty.resetDeviceConfigForPackages(
998                 Arrays.asList(new String[]{CALLING_PACKAGE1, CALLING_PACKAGE2}));
999         ArraySet<String> expectedNamespacesWiped = new ArraySet<String>(
1000                 Arrays.asList(new String[]{NAMESPACE1, NAMESPACE2, NAMESPACE3}));
1001         assertEquals(mNamespacesWiped, expectedNamespacesWiped);
1002     }
1003 
1004     @Test
testResetDeviceConfigNoExceptionWhenFlagMalformed()1005     public void testResetDeviceConfigNoExceptionWhenFlagMalformed() {
1006         RescueParty.onSettingsProviderPublished(mMockContext);
1007         verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
1008                 any(Executor.class),
1009                 mMonitorCallbackCaptor.capture()));
1010 
1011         // Record DeviceConfig accesses
1012         RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
1013         DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
1014         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
1015         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE2, NAMESPACE3);
1016         monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE3, NAMESPACE4);
1017         // Fake DeviceConfig value changes
1018         monitorCallback.onNamespaceUpdate(NAMESPACE1);
1019         monitorCallback.onNamespaceUpdate(NAMESPACE2);
1020         monitorCallback.onNamespaceUpdate(NAMESPACE3);
1021         monitorCallback.onNamespaceUpdate(NAMESPACE4);
1022 
1023         String invalidPresetMapping = NAMESPACE2 + ":" + CALLING_PACKAGE2 + ","
1024                 + NAMESPACE1 + "." + CALLING_PACKAGE2;
1025         doReturn(invalidPresetMapping).when(() -> DeviceConfig.getString(
1026                 eq(RescueParty.NAMESPACE_CONFIGURATION),
1027                 eq(RescueParty.NAMESPACE_TO_PACKAGE_MAPPING_FLAG),
1028                 eq("")));
1029 
1030         RescueParty.resetDeviceConfigForPackages(
1031                 Arrays.asList(new String[]{CALLING_PACKAGE1, CALLING_PACKAGE2}));
1032         ArraySet<String> expectedNamespacesWiped = new ArraySet<String>(
1033                 Arrays.asList(new String[]{NAMESPACE1, NAMESPACE3}));
1034         assertEquals(mNamespacesWiped, expectedNamespacesWiped);
1035     }
1036 
verifySettingsResets(int resetMode, String[] resetNamespaces, HashMap<String, Integer> configResetVerifiedTimesMap)1037     private void verifySettingsResets(int resetMode, String[] resetNamespaces,
1038             HashMap<String, Integer> configResetVerifiedTimesMap) {
1039         verifyOnlySettingsReset(resetMode);
1040         verifyDeviceConfigReset(resetNamespaces, configResetVerifiedTimesMap);
1041     }
1042 
verifyOnlySettingsReset(int resetMode)1043     private void verifyOnlySettingsReset(int resetMode) {
1044         verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
1045                 resetMode, UserHandle.USER_SYSTEM));
1046         verify(() -> Settings.Secure.resetToDefaultsAsUser(eq(mMockContentResolver), isNull(),
1047                 eq(resetMode), anyInt()));
1048     }
1049 
verifyNoSettingsReset(int resetMode)1050     private void verifyNoSettingsReset(int resetMode) {
1051         verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
1052                 resetMode, UserHandle.USER_SYSTEM), never());
1053         verify(() -> Settings.Secure.resetToDefaultsAsUser(eq(mMockContentResolver), isNull(),
1054                 eq(resetMode), anyInt()), never());
1055     }
1056 
verifyDeviceConfigReset(String[] resetNamespaces, Map<String, Integer> configResetVerifiedTimesMap)1057     private void verifyDeviceConfigReset(String[] resetNamespaces,
1058             Map<String, Integer> configResetVerifiedTimesMap) {
1059         if (resetNamespaces == null) {
1060             verify(() -> DeviceConfig.resetToDefaults(anyInt(), anyString()), never());
1061         } else {
1062             for (String namespace : resetNamespaces) {
1063                 int verifiedTimes = 0;
1064                 if (configResetVerifiedTimesMap != null
1065                         && configResetVerifiedTimesMap.get(namespace) != null) {
1066                     verifiedTimes = configResetVerifiedTimesMap.get(namespace);
1067                 }
1068                 verify(() -> DeviceConfig.resetToDefaults(RescueParty.DEVICE_CONFIG_RESET_MODE,
1069                         namespace), times(verifiedTimes + 1));
1070                 if (configResetVerifiedTimesMap != null) {
1071                     configResetVerifiedTimesMap.put(namespace, verifiedTimes + 1);
1072                 }
1073             }
1074         }
1075     }
1076 
noteBoot(int mitigationCount)1077     private void noteBoot(int mitigationCount) {
1078         RescuePartyObserver.getInstance(mMockContext).executeBootLoopMitigation(mitigationCount);
1079     }
1080 
noteAppCrash(int mitigationCount, boolean isPersistent)1081     private void noteAppCrash(int mitigationCount, boolean isPersistent) {
1082         String packageName = isPersistent ? PERSISTENT_PACKAGE : NON_PERSISTENT_PACKAGE;
1083         RescuePartyObserver.getInstance(mMockContext).execute(new VersionedPackage(
1084                 packageName, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, mitigationCount);
1085     }
1086 
1087     // Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
mockCrashRecoveryProperties(PackageWatchdog watchdog)1088     private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
1089         // mock properties in RescueParty
1090         try {
1091 
1092             doAnswer((Answer<Boolean>) invocationOnMock -> {
1093                 String storedValue = mCrashRecoveryPropertiesMap
1094                         .getOrDefault("crashrecovery.attempting_factory_reset", "false");
1095                 return Boolean.parseBoolean(storedValue);
1096             }).when(() -> RescueParty.isFactoryResetPropertySet());
1097             doAnswer((Answer<Void>) invocationOnMock -> {
1098                 boolean value = invocationOnMock.getArgument(0);
1099                 mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_factory_reset",
1100                         Boolean.toString(value));
1101                 return null;
1102             }).when(() -> RescueParty.setFactoryResetProperty(anyBoolean()));
1103 
1104             doAnswer((Answer<Boolean>) invocationOnMock -> {
1105                 String storedValue = mCrashRecoveryPropertiesMap
1106                         .getOrDefault("crashrecovery.attempting_reboot", "false");
1107                 return Boolean.parseBoolean(storedValue);
1108             }).when(() -> RescueParty.isRebootPropertySet());
1109             doAnswer((Answer<Void>) invocationOnMock -> {
1110                 boolean value = invocationOnMock.getArgument(0);
1111                 setCrashRecoveryPropAttemptingReboot(value);
1112                 return null;
1113             }).when(() -> RescueParty.setRebootProperty(anyBoolean()));
1114 
1115             doAnswer((Answer<Long>) invocationOnMock -> {
1116                 String storedValue = mCrashRecoveryPropertiesMap
1117                         .getOrDefault("persist.crashrecovery.last_factory_reset", "0");
1118                 return Long.parseLong(storedValue);
1119             }).when(() -> RescueParty.getLastFactoryResetTimeMs());
1120             doAnswer((Answer<Void>) invocationOnMock -> {
1121                 long value = invocationOnMock.getArgument(0);
1122                 setCrashRecoveryPropLastFactoryReset(value);
1123                 return null;
1124             }).when(() -> RescueParty.setLastFactoryResetTimeMs(anyLong()));
1125 
1126             doAnswer((Answer<Integer>) invocationOnMock -> {
1127                 String storedValue = mCrashRecoveryPropertiesMap
1128                         .getOrDefault("crashrecovery.max_rescue_level_attempted", "0");
1129                 return Integer.parseInt(storedValue);
1130             }).when(() -> RescueParty.getMaxRescueLevelAttempted());
1131             doAnswer((Answer<Void>) invocationOnMock -> {
1132                 int value = invocationOnMock.getArgument(0);
1133                 mCrashRecoveryPropertiesMap.put("crashrecovery.max_rescue_level_attempted",
1134                         Integer.toString(value));
1135                 return null;
1136             }).when(() -> RescueParty.setMaxRescueLevelAttempted(anyInt()));
1137 
1138         } catch (Exception e) {
1139             // tests will fail, just printing the error
1140             System.out.println("Error while mocking crashrecovery properties " + e.getMessage());
1141         }
1142 
1143         // mock properties in BootThreshold
1144         try {
1145             mSpyBootThreshold = spy(watchdog.new BootThreshold(
1146                     PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
1147                     PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
1148             mCrashRecoveryPropertiesMap = new HashMap<>();
1149 
1150             doAnswer((Answer<Integer>) invocationOnMock -> {
1151                 String storedValue = mCrashRecoveryPropertiesMap
1152                         .getOrDefault("crashrecovery.rescue_boot_count", "0");
1153                 return Integer.parseInt(storedValue);
1154             }).when(mSpyBootThreshold).getCount();
1155             doAnswer((Answer<Void>) invocationOnMock -> {
1156                 int count = invocationOnMock.getArgument(0);
1157                 setCrashRecoveryPropRescueBootCount(count);
1158                 return null;
1159             }).when(mSpyBootThreshold).setCount(anyInt());
1160 
1161             doAnswer((Answer<Integer>) invocationOnMock -> {
1162                 String storedValue = mCrashRecoveryPropertiesMap
1163                         .getOrDefault("crashrecovery.boot_mitigation_count", "0");
1164                 return Integer.parseInt(storedValue);
1165             }).when(mSpyBootThreshold).getMitigationCount();
1166             doAnswer((Answer<Void>) invocationOnMock -> {
1167                 int count = invocationOnMock.getArgument(0);
1168                 mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_count",
1169                         Integer.toString(count));
1170                 return null;
1171             }).when(mSpyBootThreshold).setMitigationCount(anyInt());
1172 
1173             doAnswer((Answer<Long>) invocationOnMock -> {
1174                 String storedValue = mCrashRecoveryPropertiesMap
1175                         .getOrDefault("crashrecovery.rescue_boot_start", "0");
1176                 return Long.parseLong(storedValue);
1177             }).when(mSpyBootThreshold).getStart();
1178             doAnswer((Answer<Void>) invocationOnMock -> {
1179                 long count = invocationOnMock.getArgument(0);
1180                 mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_start",
1181                         Long.toString(count));
1182                 return null;
1183             }).when(mSpyBootThreshold).setStart(anyLong());
1184 
1185             doAnswer((Answer<Long>) invocationOnMock -> {
1186                 String storedValue = mCrashRecoveryPropertiesMap
1187                         .getOrDefault("crashrecovery.boot_mitigation_start", "0");
1188                 return Long.parseLong(storedValue);
1189             }).when(mSpyBootThreshold).getMitigationStart();
1190             doAnswer((Answer<Void>) invocationOnMock -> {
1191                 long count = invocationOnMock.getArgument(0);
1192                 mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_start",
1193                         Long.toString(count));
1194                 return null;
1195             }).when(mSpyBootThreshold).setMitigationStart(anyLong());
1196 
1197             Field mBootThresholdField = watchdog.getClass().getDeclaredField("mBootThreshold");
1198             mBootThresholdField.setAccessible(true);
1199             mBootThresholdField.set(watchdog, mSpyBootThreshold);
1200         } catch (Exception e) {
1201             // tests will fail, just printing the error
1202             System.out.println("Error while spying BootThreshold " + e.getMessage());
1203         }
1204     }
1205 
setCrashRecoveryPropRescueBootCount(int count)1206     private void setCrashRecoveryPropRescueBootCount(int count) {
1207         mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
1208                 Integer.toString(count));
1209     }
1210 
setCrashRecoveryPropAttemptingReboot(boolean value)1211     private void setCrashRecoveryPropAttemptingReboot(boolean value) {
1212         mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_reboot",
1213                 Boolean.toString(value));
1214     }
1215 
setCrashRecoveryPropLastFactoryReset(long value)1216     private void setCrashRecoveryPropLastFactoryReset(long value) {
1217         mCrashRecoveryPropertiesMap.put("persist.crashrecovery.last_factory_reset",
1218                 Long.toString(value));
1219     }
1220 }
1221