[ { description: "some AlarmClockTests are not robust across different device types", names: [ "android.alarmclock.cts.DismissAlarmTest#testAll", "android.alarmclock.cts.SetAlarmTest#testAll", "android.alarmclock.cts.SnoozeAlarmTest#testAll" ], bug: 23776083 }, { description: "the UsageStats is not yet stable enough", names: [ "android.app.usage.cts.UsageStatsTest" ], bug: 17536113 }, { description: "the ConnectivityConstraintTest are not yet stable", names: [ "android.jobscheduler.cts.ConnectivityConstraintTest" ], bug: 18117279 }, { description: "tests a fragile by nature as they rely on hardcoded behavior", names: [ "android.accessibilityservice.cts.AccessibilityTextTraversalTest#testActionNextAndPreviousAtGranularityPageOverText", "android.accessibilityservice.cts.AccessibilityTextTraversalTest#testActionNextAndPreviousAtGranularityPageOverTextExtend" ], bug: 17595050 }, { description: "test fails on some devices", names: [ "android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput", "android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats" ], bug: 23776893 }, { description: "the SSLCertificateSocketFactoryTest often fails because of lack of live internet or short timeout, it should be refactored to do a local server testing", names: [ "android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket", "android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_bind", "android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_simple", "android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping" ], bug: 18682315 }, { description: "the test result are too much dependent on live-internet connection, which for some devices might not exist", names: [ "android.net.wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly" ], bug: 18680089 }, { description: "AudioPolicyBinder tests are not yet robust enough", names: [ "android.security.cts.AudioPolicyBinderTest" ], bug: 18461670 }, { description: "test not robust", names: [ "android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect", "android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections" ], bug: 23604254 }, { description: "tests too flaky", names: [ "android.transition.cts.ChangeScrollTest#testChangeScroll" ], bug: 23779020 }, { description: "Not all jdwp features are currently supported. These tests will fail", names: [ "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch001", "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch002", "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch003", "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch004", "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowLaunchDebugger001#testDebugger002", "org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowLaunchDebugger002#testDebugger", "org.apache.harmony.jpda.tests.jdwp.Events.ClassUnloadTest#testClassUnloadEvent", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorContendedEnterTest#testMonitorContendedEnterForClassMatch", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorContendedEnteredTest#testMonitorContendedEnteredForClassMatch", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassExclude", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassMatchExact", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassMatchFirst", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassMatchSecond", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitTest#testMonitorWaitForClassOnly", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassExclude", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassMatchExact", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassMatchFirst", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassMatchSecond", "org.apache.harmony.jpda.tests.jdwp.Events.MonitorWaitedTest#testMonitorWaitedForClassOnly", "org.apache.harmony.jpda.tests.jdwp.ReferenceType.ClassFileVersionTest#testClassFileVersion001", "org.apache.harmony.jpda.tests.jdwp.ReferenceType.NestedTypesTest#testNestedTypes001", "org.apache.harmony.jpda.tests.jdwp.ThreadReference.StopTest#testStop001", "org.apache.harmony.jpda.tests.jdwp.VirtualMachine.HoldEventsTest#testHoldEvents001", "org.apache.harmony.jpda.tests.jdwp.VirtualMachine.ReleaseEventsTest#testReleaseEvents001" ], bug: 16720689 }, { description: "test can only run properly on a user build device when the bug is resolved", names: [ "android.appwidget.cts.AppWidgetTest#testAppWidgetProviderCallbacks", "android.appwidget.cts.AppWidgetTest#testBindAppWidget", "android.appwidget.cts.AppWidgetTest#testCollectionWidgets", "android.appwidget.cts.AppWidgetTest#testDeleteHost", "android.appwidget.cts.AppWidgetTest#testDeleteHosts", "android.appwidget.cts.AppWidgetTest#testGetAppWidgetIds", "android.appwidget.cts.AppWidgetTest#testGetAppWidgetInfo", "android.appwidget.cts.AppWidgetTest#testGetAppWidgetOptions", "android.appwidget.cts.AppWidgetTest#testPartiallyUpdateAppWidgetViaWidgetId", "android.appwidget.cts.AppWidgetTest#testPartiallyUpdateAppWidgetViaWidgetIds", "android.appwidget.cts.AppWidgetTest#testTwoAppWidgetProviderCallbacks", "android.appwidget.cts.AppWidgetTest#testUpdateAppWidgetViaComponentName", "android.appwidget.cts.AppWidgetTest#testUpdateAppWidgetViaWidgetId", "android.appwidget.cts.AppWidgetTest#testUpdateAppWidgetViaWidgetIds" ], bug: 17993121 }, { description: "permissions for the API previously used in the test has changed, making it impossible to pass", names: [ "android.openglperf.cts.GlAppSwitchTest#testGlActivitySwitchingFast", "android.openglperf.cts.GlAppSwitchTest#testGlActivitySwitchingSlow" ], bug: 17394321 }, { description: "unexpected failures", names: [ "android.openglperf.cts.GlVboPerfTest#testVboWithVaryingIndexBufferNumbers" ], bug: 18091590 }, { description: "these tests require a good test scene, so they fail if run in random conditions", names: [ "android.hardware.camera2.cts.AllocationTest#testBlackWhite", "android.hardware.camera2.cts.AllocationTest#testParamSensitivity" ], bug: 17530117 }, { description: "Current implementation of uninstallAllUserCaCerts does not throw expected security exception, wait for fix from framework", names: [ "android.admin.cts.DevicePolicyManagerTest#testUninstallAllUserCaCerts_failIfNotProfileOwner" ], bug: 17508787 }, { description: "Test is not yet properly implemented", names: [ "android.voicesettings.cts.ZenModeTest#testAll" ], bug: 23238984 }, { description: "This test failed on devices that use effect off loading. In addition it uses hidden apis", names: [ "android.media.cts.AudioEffectTest#test1_1ConstructorFromUuid" ], bug: 17605875 }, { description: "This test failed on hw decoder that doesn't output frame with the configured format.", names: [ "android.media.cts.ImageReaderDecoderTest#testHwAVCDecode360pForFlexibleYuv" ], bug: 17144778 }, { description: "android.keystore tests will replace these tests", names: [ "com.android.org.conscrypt.MacTest#test_getInstance_OpenSSL_ENGINE", "com.android.org.conscrypt.NativeCryptoTest#test_ENGINE_by_id_TestEngine", "com.android.org.conscrypt.SignatureTest#test_getInstance_OpenSSL_ENGINE" ], bug: 18030049 }, { description: "The new prepare performance test is not yet passing on all devices", names: [ "android.hardware.camera2.cts.SurfaceViewPreviewTest#testPreparePerformance" ], bug: 17989532 }, { description: "The timing measurements for preview callbacks are not reliable", names: [ "android.hardware.cts.CameraTest#testPreviewFpsRange" ], bug: 23008511 }, { description: "Light status bar CTS coming in late", names: [ "android.systemui.cts.LightStatusBarTests#testLightStatusBarIcons" ], bug: 23427621 }, { description: "known failures", names: [ "android.hardware.cts.SensorBatchingTests#testAccelerometer_50hz_batching", "android.hardware.cts.SensorBatchingTests#testAccelerometer_fastest_batching", "android.hardware.cts.SensorBatchingTests#testGyroscope_50hz_batching", "android.hardware.cts.SensorBatchingTests#testGyroscope_50hz_flush", "android.hardware.cts.SensorBatchingTests#testMagneticField_50hz_batching", "android.hardware.cts.SensorBatchingTests#testMagneticField_fastest_batching", "android.hardware.cts.SensorBatchingTests#testMagneticFieldUncalibrated_50hz_batching", "android.hardware.cts.SensorBatchingTests#testMagneticFieldUncalibrated_fastest_batching", "android.hardware.cts.SensorBatchingTests#testPressure_50hz_batching", "android.hardware.cts.SensorBatchingTests#testPressure_fastest_batching", "android.hardware.cts.SensorBatchingTests#testRotationVector_50hz_flush", "android.hardware.cts.SensorBatchingTests#testRotationVector_fastest_batching", "android.hardware.cts.SensorIntegrationTests#testSensorsMovingRates", "android.hardware.cts.SensorIntegrationTests#testSensorsWithSeveralClients", "android.hardware.cts.SensorTest#testSensorTimeStamps", "android.hardware.cts.SensorTest#testBatchAndFlush", "android.hardware.cts.SingleSensorTests#testGyroscope_15hz", "android.hardware.cts.SingleSensorTests#testGyroscope_1hz", "android.hardware.cts.SingleSensorTests#testMagneticField_1hz", "android.hardware.cts.SingleSensorTests#testMagneticField_50hz", "android.hardware.cts.SingleSensorTests#testMagneticFieldUncalibrated_200hz", "android.hardware.cts.SingleSensorTests#testOrientation_5hz" ], bug: 22922206 }, { description: "tests are not yet ready", names: [ "com.android.cts.app.os.OsHostTests#testNonExportedActivities" ], bug: 23779168 }, { description: "New assist tests that do not yet have a track record.", names: [ "android.assist.cts.AssistantContentViewTest", "android.assist.cts.ExtraAssistDataTest", "android.assist.cts.FocusChangeTest", "android.assist.cts.LargeViewHierarchyTest", "android.assist.cts.ScreenshotTest", "android.assist.cts.TextViewTest", "android.assist.cts.WebViewTest" ], bug: 21668302 }, { description: "ConnectivityConstraintTest job scheduler not working.", names: [ "android.jobscheduler.cts.ConnectivityConstraintTest#testConnectivityConstraintExecutes_withWifi", "android.jobscheduler.cts.ConnectivityConstraintTest#testUnmeteredConstraintExecutes_withWifi", "android.jobscheduler.cts.ConnectivityConstraintTest#testConnectivityConstraintExecutes_withMobile" ], bug: 21262226 }, { description: "ConnectivityConstraintTest times out.", names: [ "android.jobscheduler.cts.TimingConstraintsTest#testJobParameters_unexpiredDeadline" ], bug: 23144425 }, { description: "Telephony returning wrong value.", names: [ "android.telephony.cts.CellInfoTest#testCellInfo" ], bug: 23979591 }, { description: "Video encoding tests are timing out.", names: [ "android.media.cts.VideoEncoderTest#testGoogH264FlexArbitraryW", "android.media.cts.VideoEncoderTest#testGoogH264SurfArbitraryW" ], bug: 23827982 }, { description: "protected broadcast not working", names: [ "android.permission2.cts.ProtectedBroadcastsTest#testSendProtectedBroadcasts" ], bug: 23192492 }, { description: "restricted network is not working", names: [ "android.net.cts.ConnectivityManagerTest#testRestrictedNetworks" ], bug: 25651805 }, { description: "unit testing for MediaPreparer lives within mediastress module", names: [ "android.mediastress.cts.preconditions.MediaPreparerTest" ], bug: 25850508 }, { description: "Tests for the signature tests should not be in CTS", names: [ "android.signature.cts.tests" ], bug: 26150806 }, { description: "android.security.cts is using a Non-NDK library, libmedia_jni.so", names: [ "android.security.cts.MediaCryptoTest#testMediaCryptoClearKey", "android.security.cts.MediaCryptoTest#testMediaCryptoWidevine" ], bug: 27218502 }, { description: "Still investigating this, root cause unknown yet", bug: 27578806, names: ["com.android.cts.cpptools.RunAsHostTest#testRunAs"] } ]