1/*
2 * This file contains expectations for ART's buildbot. The purpose of this file is
3 * to temporarily list failing tests and not break the bots, until the
4 * libcore expectation files get properly updated. The script that uses this file
5 * is art/tools/run-libcore-tests.sh.
6 *
7 * It is also used to enable AOSP experiments, and not mess up with CTS's expectations.
8 */
9
10[
11{
12  description: "Os.memfd_create() only supports bionic-based platforms.",
13  result: EXEC_FAILED,
14  modes: [host],
15  names: ["libcore.android.system.OsTest#testMemfdCreate",
16          "libcore.android.system.OsTest#testMemfdCreateErrno",
17          "libcore.android.system.OsTest#testMemfdCreateFlags"]
18},
19{
20  description: "Assert.java differences between vogar and junit.",
21  result: EXEC_FAILED,
22  modes: [host],
23  name: "libcore.java.math.RunCSVTests#test_csv"
24},
25{
26  description: "Differences between vogar and cts in user directory",
27  result: EXEC_FAILED,
28  modes: [device_testdex],
29  name: "libcore.java.lang.SystemTest#testSystemProperties_mutable"
30},
31{
32  description: "Differences between vogar and cts. Passes with --mode activity",
33  result: EXEC_FAILED,
34  modes: [device_testdex],
35  names: ["libcore.java.lang.OldSystemTest#test_getProperties"]
36},
37{
38  description: "Differences between vogar and cts. EACCESS when run with vogar.
39                Passes on host, passes with cts. Passes with vogar with su
40                (--invoke-with \"su root\"). Does not pass after setting chmod
41                777 all directories on path to socket (on device without su).",
42  result: EXEC_FAILED,
43  modes: [device_testdex],
44  names: ["libcore.libcore.io.OsTest#testUnixDomainSockets_in_file_system"]
45},
46{
47  description: "TCP_USER_TIMEOUT is not defined on host's tcp.h (glibc-2.15-4.8).",
48  result: EXEC_FAILED,
49  modes: [host],
50  names: ["libcore.android.system.OsConstantsTest#testTcpUserTimeoutIsDefined",
51          "libcore.libcore.io.OsTest#test_socket_tcpUserTimeout_setAndGet",
52          "libcore.libcore.io.OsTest#test_socket_tcpUserTimeout_doesNotWorkOnDatagramSocket"],
53  bug: 30402085
54},
55{
56  description: "Issue with incorrect device time (1970)",
57  result: EXEC_FAILED,
58  modes: [device_testdex],
59  names: ["libcore.java.util.TimeZoneTest#testDisplayNames",
60          "libcore.java.util.TimeZoneTest#test_useDaylightTime_Taiwan",
61          "org.apache.harmony.tests.java.util.TimeZoneTest#test_hasSameRules_Ljava_util_TimeZone"],
62  bug: 20879084
63},
64{
65  description: "Issue with incorrect device time (1970). Test assumes that DateTime.now()
66                is greater then a date in 1998.",
67  result: EXEC_FAILED,
68  modes: [device_testdex],
69  names: ["org.apache.harmony.tests.java.util.DateTest#test_Constructor"]
70},
71{
72  description: "Failing due to a locale problem on hammerhead.",
73  result: EXEC_FAILED,
74  modes: [device_testdex],
75  names: ["libcore.icu.DateIntervalFormatTest#test10089890",
76          "libcore.icu.DateIntervalFormatTest#test10209343_when_not_this_year",
77          "libcore.icu.DateIntervalFormatTest#test10560853_for_single_day_events",
78          "libcore.icu.DateIntervalFormatTest#test10560853_when_time_not_displayed",
79          "libcore.icu.RelativeDateTimeFormatterTest#test_getRelativeDateTimeString",
80          "libcore.icu.RelativeDateTimeFormatterTest#test_getRelativeTimeSpanString",
81          "libcore.icu.RelativeDateTimeFormatterTest#test_getRelativeTimeSpanStringAbbrev",
82          "libcore.java.text.OldDateFormatTest#test_parseLString",
83          "libcore.java.text.SimpleDateFormatTest#testDstZoneNameWithNonDstTimestamp",
84          "libcore.java.text.SimpleDateFormatTest#testDstZoneWithNonDstTimestampForNonHourDstZone",
85          "libcore.java.text.SimpleDateFormatTest#testNonDstZoneNameWithDstTimestamp",
86          "libcore.java.text.SimpleDateFormatTest#testNonDstZoneWithDstTimestampForNonHourDstZone"]
87},
88{
89  description: "Failing due to missing localhost on hammerhead and volantis.",
90  result: EXEC_FAILED,
91  modes: [device_testdex],
92  names: ["libcore.javax.crypto.CipherTest#testCipherInitWithCertificate",
93          "libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithFtpURLConnection",
94          "libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithJarFtpURLConnection",
95          "libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithLoggingSocketHandler",
96          "libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithHttpURLConnection",
97          "org.apache.harmony.luni.tests.internal.net.www.protocol.http.HttpURLConnectionTest",
98          "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnectionTest",
99          "org.apache.harmony.luni.tests.java.net.URLConnectionTest",
100          "org.apache.harmony.tests.java.util.ScannerTest#test_Constructor_LReadableByteChannel"
101         ]
102},
103{
104  description: "Test timeouts",
105  result: EXEC_TIMEOUT,
106  modes: [device_testdex],
107  names: ["org.apache.harmony.tests.java.util.ScannerTest#testPerformance"]
108},
109{
110  description: "Needs the newest cat version on the device",
111  result: EXEC_FAILED,
112  modes: [device_testdex],
113  names: ["org.apache.harmony.tests.java.lang.ProcessTest#test_getErrorStream"]
114},
115{
116  description: "Needs kernel updates on host/device",
117  result: EXEC_FAILED,
118  names: ["libcore.libcore.io.OsTest#test_socketPing"]
119},
120{
121  description: "Linker issues in chrooted environment",
122  modes: [device_testdex],
123  result: EXEC_FAILED,
124  names: ["org.apache.harmony.tests.java.lang.ProcessManagerTest#testEnvironment"]
125},
126{
127  description: "Flake when running with libartd.so or interpreter",
128  result: EXEC_FAILED,
129  bug:22106064,
130  name: "libcore.java.lang.OldThreadGroupTest#test_enumerateLThreadArrayLZtest_enumerateLThreadArrayLZ"
131},
132{
133  description: "test_xattr fails on arm64 on the buildbots only: needs investigation",
134  result: EXEC_FAILED,
135  modes: [device_testdex],
136  names: ["libcore.libcore.io.OsTest#test_xattr"],
137  bug: 22258911
138},
139{
140  description: "fails on L builds: needs investigation",
141  result: EXEC_FAILED,
142  modes: [device_testdex],
143  names: ["org.apache.harmony.tests.java.lang.ClassTest#test_forNameLjava_lang_String"]
144},
145{
146  description: "Lack of IPv6 on some buildbot slaves",
147  result: EXEC_FAILED,
148  names: ["libcore.libcore.io.OsTest#test_byteBufferPositions_sendto_recvfrom_af_inet6",
149          "libcore.libcore.io.OsTest#test_sendtoSocketAddress_af_inet6",
150          "libcore.libcore.io.OsTest#test_recvfrom_EmptyPacket"],
151  bug: 25178637
152},
153{
154  description: "Non-deterministic test because of a dependency on weak ref collection.",
155  result: EXEC_FAILED,
156  names: ["org.apache.harmony.tests.java.util.WeakHashMapTest#test_keySet"],
157  bug: 25437292
158},
159{
160  description: "Only work with --mode=activity",
161  result: EXEC_FAILED,
162  names: [ "libcore.java.io.FileTest#testJavaIoTmpdirMutable" ]
163},
164{
165  description: "Flaky test",
166  result: EXEC_FAILED,
167  bug: 30107038,
168  modes: [device_testdex],
169  names: ["org.apache.harmony.tests.java.lang.ProcessTest#test_destroyForcibly"]
170},
171{
172  description: "Flaky failure, native crash in the runtime.
173                Unclear if this relates to the tests running sh as a child process.",
174  result: EXEC_FAILED,
175  bug: 30657148,
176  modes: [device_testdex],
177  names: ["libcore.java.lang.ProcessBuilderTest#testRedirectInherit",
178          "libcore.java.lang.ProcessBuilderTest#testRedirect_nullStreams"]
179},
180{
181  description: "java.io.IOException: Error writing ASN.1 encoding",
182  result: EXEC_FAILED,
183  names: ["libcore.javax.crypto.spec.AlgorithmParametersTestGCM#testEncoding"]
184},
185{
186  description: "fdsan doesn't exist on the host",
187  result: EXEC_FAILED,
188  modes: [host],
189  bug: 113177877,
190  names: ["libcore.libcore.io.FdsanTest#testFileInputStream",
191          "libcore.libcore.io.FdsanTest#testFileOutputStream",
192          "libcore.libcore.io.FdsanTest#testRandomAccessFile",
193          "libcore.libcore.io.FdsanTest#testParcelFileDescriptor",
194          "libcore.libcore.io.FdsanTest#testDatagramSocket",
195          "libcore.libcore.io.FdsanTest#testSocket"]
196},
197{
198  description: "Timeout on heap-poisoning target builds",
199  result: EXEC_FAILED,
200  modes: [device_testdex],
201  bug: 116446372,
202  names: ["libcore.libcore.io.FdsanTest#testSocket"]
203},
204{
205  description: "Host implementation of android_getaddrinfo differs from device implementation",
206  result: EXEC_FAILED,
207  modes: [host],
208  bug: 121230364,
209  names: [
210    "libcore.libcore.net.InetAddressUtilsTest#parseNumericAddress[8]",
211    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[10]",
212    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[11]",
213    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[12]",
214    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[5]",
215    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[6]",
216    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[7]",
217    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[8]",
218    "libcore.libcore.net.InetAddressUtilsTest#test_isNotNumericAddress[9]",
219    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[10]",
220    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[11]",
221    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[12]",
222    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[5]",
223    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[6]",
224    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[7]",
225    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[8]",
226    "libcore.libcore.net.InetAddressUtilsTest#test_parseNonNumericAddress[9]"
227  ]
228},
229{
230  description: "Apex related",
231  result: EXEC_FAILED,
232  modes: [device_testdex],
233  bug: 122642227,
234  names: [
235    "libcore.libcore.icu.TimeZoneIntegrationTest#testTimeZoneDataVersion",
236    "libcore.libcore.icu.TimeZoneIntegrationTest#testTimeZoneDebugInfo",
237    "libcore.libcore.icu.TimeZoneIntegrationTest#testTzDataSetVersions"
238  ]
239},
240{
241  description: "Timeout on walleye bots, currenlty on OPM1.171019.019. But test passes on Q, so just mark it fail here.",
242  result: EXEC_FAILED,
243  modes: [device_testdex],
244  names: [
245    "libcore.libcore.io.OsTest#testCrossFamilyBindConnectSendto"
246  ]
247},
248{
249  description: "Test fails on host with `socket failed: EACCES (Permission denied)`",
250  result: EXEC_FAILED,
251  modes: [host],
252  names: ["libcore.android.system.OsTest#test_socketPing"]
253},
254{
255  description: "Test fails on host with: `/home/dalvik-prebuild/jaxen/xml/test/tests.xml: open failed: ENOENT (No such file or directory)`",
256  result: EXEC_FAILED,
257  names: ["libcore.xml.JaxenXPathTestSuite#suite"]
258},
259{
260  description: "Test fails on host with: `/home/dalvik-prebuild/OASIS/XSLT-Conformance-TC/TESTS/catalog.xml: open failed: ENOENT (No such file or directory)`",
261  result: EXEC_FAILED,
262  names: ["libcore.xml.XsltXPathConformanceTestSuite#suite"]
263},
264{
265  description: "Fails on device",
266  result: EXEC_FAILED,
267  modes: [device_testdex],
268  bug: 145348591,
269  names: ["libcore.android.system.OsTest#test_if_nametoindex_if_indextoname"]
270},
271{
272  description: "Fails on non-Android and Android versions < R",
273  result: EXEC_FAILED,
274  names: ["libcore.android.system.OsTest#test_NetlinkSocket"]
275}
276]
277