1[
2{
3  description: "some AlarmClockTests are not robust across different device types",
4  names: [
5    "android.alarmclock.cts.DismissAlarmTest#testAll",
6    "android.alarmclock.cts.SetAlarmTest#testAll",
7    "android.alarmclock.cts.SnoozeAlarmTest#testAll"
8  ],
9  bug: 23776083
10},
11{
12  description: "the UsageStats is not yet stable enough",
13  names: [
14    "android.app.usage.cts.UsageStatsTest"
15  ],
16  bug: 17536113
17},
18{
19  description: "tests a fragile by nature as they rely on hardcoded behavior",
20  names: [
21    "android.accessibilityservice.cts.AccessibilityTextTraversalTest#testActionNextAndPreviousAtGranularityPageOverText",
22    "android.accessibilityservice.cts.AccessibilityTextTraversalTest#testActionNextAndPreviousAtGranularityPageOverTextExtend"
23  ],
24  bug: 17595050
25},
26{
27  description: "test fails on devices with no telephony",
28  names: [
29    "android.calllog.cts.CallLogBackupTest#testSingleCallBackup"
30  ],
31  bug: 23776099
32},
33{
34  description: "test fails on some devices",
35  names: [
36    "android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput",
37    "android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats"
38  ],
39  bug: 23776893
40},
41{
42  description: "the SSLCertificateSocketFactoryTest often fails because of lack of live internet or short timeout, it should be refactored to do a local server testing",
43  names: [
44    "android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket",
45    "android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_bind",
46    "android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_simple",
47    "android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping"
48  ],
49  bug: 18682315
50},
51{
52  description: "the test result are too much dependent on live-internet connection, which for some devices might not exist",
53  names: [
54    "android.net.wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly"
55  ],
56  bug: 18680089
57},
58{
59  description: "AudioPolicyBinder tests are not yet robust enough",
60  names: [
61    "android.security.cts.AudioPolicyBinderTest"
62  ],
63  bug: 18461670
64},
65{
66  description: "test not robust",
67  names: [
68    "android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect",
69    "android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections"
70  ],
71  bug: 23604254
72},
73{
74  description: "tests too flaky",
75  names: [
76    "android.transition.cts.ChangeScrollTest#testChangeScroll"
77  ],
78  bug: 23779020
79},
80{
81  description: "Not all jdwp features are currently supported. These tests will fail",
82  names: [
83    "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch001",
84    "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch002",
85    "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch003",
86    "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch004",
87    "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowLaunchDebugger001#testDebugger002",
88    "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowLaunchDebugger002#testDebugger",
89    "org.apache.harmony.jpda.tests.jdwp.Events.ClassUnloadTest#testClassUnloadEvent",
90    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorContendedEnterTest#testMonitorContendedEnterForClassMatch",
91    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorContendedEnteredTest#testMonitorContendedEnteredForClassMatch",
92    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassExclude",
93    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassMatchExact",
94    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassMatchFirst",
95    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassMatchSecond",
96    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassOnly",
97    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassExclude",
98    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassMatchExact",
99    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassMatchFirst",
100    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassMatchSecond",
101    "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassOnly",
102    "org.apache.harmony.jpda.tests.jdwp.ReferenceType.ClassFileVersionTest#testClassFileVersion001",
103    "org.apache.harmony.jpda.tests.jdwp.ReferenceType.NestedTypesTest#testNestedTypes001",
104    "org.apache.harmony.jpda.tests.jdwp.ThreadReference.StopTest#testStop001",
105    "org.apache.harmony.jpda.tests.jdwp.VirtualMachine.HoldEventsTest#testHoldEvents001",
106    "org.apache.harmony.jpda.tests.jdwp.VirtualMachine.ReleaseEventsTest#testReleaseEvents001"
107  ],
108  bug: 16720689
109},
110{
111  description: "test can only run properly on a user build device when the bug is resolved",
112  names: [
113    "android.appwidget.cts.AppWidgetTest#testAppWidgetProviderCallbacks",
114    "android.appwidget.cts.AppWidgetTest#testBindAppWidget",
115    "android.appwidget.cts.AppWidgetTest#testCollectionWidgets",
116    "android.appwidget.cts.AppWidgetTest#testDeleteHost",
117    "android.appwidget.cts.AppWidgetTest#testDeleteHosts",
118    "android.appwidget.cts.AppWidgetTest#testGetAppWidgetIds",
119    "android.appwidget.cts.AppWidgetTest#testGetAppWidgetInfo",
120    "android.appwidget.cts.AppWidgetTest#testGetAppWidgetOptions",
121    "android.appwidget.cts.AppWidgetTest#testPartiallyUpdateAppWidgetViaWidgetId",
122    "android.appwidget.cts.AppWidgetTest#testPartiallyUpdateAppWidgetViaWidgetIds",
123    "android.appwidget.cts.AppWidgetTest#testTwoAppWidgetProviderCallbacks",
124    "android.appwidget.cts.AppWidgetTest#testUpdateAppWidgetViaComponentName",
125    "android.appwidget.cts.AppWidgetTest#testUpdateAppWidgetViaWidgetId",
126    "android.appwidget.cts.AppWidgetTest#testUpdateAppWidgetViaWidgetIds"
127  ],
128  bug: 17993121
129},
130{
131  description: "permissions for the API previously used in the test has changed, making it impossible to pass",
132  names: [
133    "android.openglperf.cts.GlAppSwitchTest#testGlActivitySwitchingFast",
134    "android.openglperf.cts.GlAppSwitchTest#testGlActivitySwitchingSlow"
135  ],
136  bug: 17394321
137},
138{
139  description: "unexpected failures",
140  names: [
141    "android.openglperf.cts.GlVboPerfTest#testVboWithVaryingIndexBufferNumbers"
142  ],
143  bug: 18091590
144},
145{
146  description: "these tests require a good test scene, so they fail if run in random conditions",
147  names: [
148    "android.hardware.camera2.cts.AllocationTest#testBlackWhite",
149    "android.hardware.camera2.cts.AllocationTest#testParamSensitivity"
150  ],
151  bug: 17530117
152},
153{
154  description: "Current implementation of uninstallAllUserCaCerts does not throw expected security exception, wait for fix from framework",
155  names: [
156    "android.admin.cts.DevicePolicyManagerTest#testUninstallAllUserCaCerts_failIfNotProfileOwner"
157  ],
158  bug: 17508787
159},
160{
161  description: "This test should be outside of official CTS suite until it is verified for all Nexus devices",
162  names: [
163    "com.android.cts.devicepolicy.MixedDeviceOwnerTest#testPackageInstallUserRestrictions",
164    "com.android.cts.devicepolicy.MixedProfileOwnerTest#testPackageInstallUserRestrictions"
165  ],
166  bug: 18928535
167},
168{
169  description: "Test is not yet properly implemented",
170  names: [
171    "android.voicesettings.cts.ZenModeTest#testAll"
172  ],
173  bug: 23238984
174},
175{
176  description: "These tests fail on some devices.",
177  names: [
178    "android.uirendering.cts.testclasses.ExactCanvasTests#testBlueRect",
179    "android.uirendering.cts.testclasses.ExactCanvasTests#testBluePaddedSquare",
180    "android.uirendering.cts.testclasses.ViewClippingTests#testSimplePaddingClip",
181    "android.uirendering.cts.testclasses.ViewClippingTests#testSimpleClipBoundsClip",
182    "android.uirendering.cts.testclasses.ViewClippingTests#testSimpleOutlineClip",
183    "android.uirendering.cts.testclasses.ViewClippingTests#testSimpleBoundsClip",
184    "android.uirendering.cts.testclasses.InfrastructureTests#testViewInitializer"
185  ],
186  bug: 17511118
187},
188{
189  description: "This test failed on devices that use effect off loading. In addition it uses hidden apis",
190  names: [
191    "android.media.cts.AudioEffectTest#test1_1ConstructorFromUuid"
192  ],
193  bug: 17605875
194},
195{
196  description: "This test failed on hw decoder that doesn't output frame with the configured format.",
197  names: [
198    "android.media.cts.ImageReaderDecoderTest#testHwAVCDecode360pForFlexibleYuv"
199  ],
200  bug: 17144778
201},
202{
203  description: "android.keystore tests will replace these tests",
204  names: [
205    "com.android.org.conscrypt.MacTest#test_getInstance_OpenSSL_ENGINE",
206    "com.android.org.conscrypt.NativeCryptoTest#test_ENGINE_by_id_TestEngine",
207    "com.android.org.conscrypt.SignatureTest#test_getInstance_OpenSSL_ENGINE"
208  ],
209  bug: 18030049
210},
211{
212  description: "The new recording test is not yet passing on all devices",
213  names: [
214    "android.hardware.camera2.cts.RecordingTest#testRecordingFramerateLowToHigh"
215  ],
216  bug: 18705837
217},
218{
219  description: "The new image reader test is not yet passing on all devices",
220  names: [
221    "android.hardware.camera2.cts.ImageReaderTest#testAllOutputYUVResolutions"
222  ],
223  bug: 18689511
224},
225{
226  description: "The new prepare performance test is not yet passing on all devices",
227  names: [
228    "android.hardware.camera2.cts.SurfaceViewPreviewTest#testPreparePerformance"
229  ],
230  bug: 17989532
231},
232{
233  description: "The new AE/AF trigger tests are not yet passing on all devices",
234  names: [
235    "android.hardware.camera2.cts.RobustnessTest#testBasicTriggerSequence",
236    "android.hardware.camera2.cts.RobustnessTest#testSimultaneousTriggers",
237    "android.hardware.camera2.cts.RobustnessTest#testAfThenAeTrigger",
238    "android.hardware.camera2.cts.RobustnessTest#testAeThenAfTrigger"
239  ],
240  bug: 22180706
241},
242{
243  description: "The new create session test is not yet passing on all devices",
244  names: [
245    "android.hardware.camera2.cts.CameraDeviceTest#testCreateSessions"
246  ],
247  bug: 22092756
248},
249{
250  description: "The new long processing test is not yet passing on all devices",
251  names: [
252    "android.hardware.camera2.cts.ImageReaderTest#testLongProcessingRepeatingRaw",
253    "android.hardware.camera2.cts.ImageReaderTest#testLongProcessingRepeatingFlexibleYuv"
254  ],
255  bug: 22861512
256},
257{
258  description: "The timing measurements for preview callbacks are not reliable",
259  names: [
260    "android.hardware.cts.CameraTest#testPreviewFpsRange"
261  ],
262  bug: 23008511
263},
264{
265  description: "known failures",
266  names: [
267    "android.hardware.cts.SensorBatchingTests#testAccelerometer_50hz_batching",
268    "android.hardware.cts.SensorBatchingTests#testAccelerometer_fastest_batching",
269    "android.hardware.cts.SensorBatchingTests#testGyroscope_50hz_batching",
270    "android.hardware.cts.SensorBatchingTests#testGyroscope_50hz_flush",
271    "android.hardware.cts.SensorBatchingTests#testMagneticField_50hz_batching",
272    "android.hardware.cts.SensorBatchingTests#testMagneticField_fastest_batching",
273    "android.hardware.cts.SensorBatchingTests#testMagneticFieldUncalibrated_50hz_batching",
274    "android.hardware.cts.SensorBatchingTests#testMagneticFieldUncalibrated_fastest_batching",
275    "android.hardware.cts.SensorBatchingTests#testPressure_50hz_batching",
276    "android.hardware.cts.SensorBatchingTests#testPressure_fastest_batching",
277    "android.hardware.cts.SensorBatchingTests#testRotationVector_50hz_flush",
278    "android.hardware.cts.SensorBatchingTests#testRotationVector_fastest_batching",
279    "android.hardware.cts.SensorIntegrationTests#testSensorsMovingRates",
280    "android.hardware.cts.SensorIntegrationTests#testSensorsWithSeveralClients",
281    "android.hardware.cts.SensorTest#testSensorTimeStamps",
282    "android.hardware.cts.SensorTest#testBatchAndFlush",
283    "android.hardware.cts.SingleSensorTests#testGyroscope_15hz",
284    "android.hardware.cts.SingleSensorTests#testGyroscope_1hz",
285    "android.hardware.cts.SingleSensorTests#testMagneticField_1hz",
286    "android.hardware.cts.SingleSensorTests#testMagneticField_50hz",
287    "android.hardware.cts.SingleSensorTests#testMagneticFieldUncalibrated_200hz",
288    "android.hardware.cts.SingleSensorTests#testOrientation_5hz"
289  ],
290  bug: 22922206
291},
292{
293  description: "tests are not yet ready",
294  names: [
295    "com.android.cts.app.os.OsHostTests#testNonExportedActivities"
296  ],
297  bug: 23779168
298},
299{
300  description: "New assist tests that do not yet have a track record.",
301  names: [
302    "android.assist.cts.ScreenshotTest",
303    "android.assist.cts.ExtraAssistDataTest",
304    "android.assist.cts.AssistantContentViewTest"
305  ],
306  bug: 21668302
307}
308]
309