1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.cts.tradefed.testtype; 17 18 import com.android.cts.tradefed.build.StubCtsBuildHelper; 19 import com.android.cts.tradefed.UnitTests; 20 import com.android.cts.util.AbiUtils; 21 import com.android.ddmlib.IDevice; 22 import com.android.ddmlib.IShellOutputReceiver; 23 import com.android.ddmlib.ShellCommandUnresponsiveException; 24 import com.android.ddmlib.testrunner.ITestRunListener; 25 import com.android.ddmlib.testrunner.TestIdentifier; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.testtype.IAbi; 30 import com.android.tradefed.util.IRunUtil; 31 import com.android.tradefed.util.RunInterruptedException; 32 33 import junit.framework.TestCase; 34 35 import org.easymock.EasyMock; 36 import org.easymock.IAnswer; 37 import org.easymock.IMocksControl; 38 39 import java.io.File; 40 import java.util.ArrayList; 41 import java.util.Collections; 42 import java.util.Collection; 43 import java.util.HashMap; 44 import java.util.List; 45 import java.util.Map; 46 import java.util.concurrent.TimeUnit; 47 48 /** 49 * Unit tests for {@link DeqpTestRunner}. 50 */ 51 public class DeqpTestRunnerTest extends TestCase { 52 private static final String NAME = "dEQP-GLES3"; 53 private static final String ID = AbiUtils.createId(UnitTests.ABI.getName(), NAME); 54 private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt"; 55 private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa"; 56 private static final String INSTRUMENTATION_NAME = 57 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 58 private static final String QUERY_INSTRUMENTATION_NAME = 59 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 60 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; 61 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; 62 private static final String ONLY_LANDSCAPE_FEATURES = 63 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE; 64 private static final String ALL_FEATURES = 65 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT; 66 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS; 67 68 static { 69 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>())70 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>()); 71 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8"); 72 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); 73 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); 74 } 75 76 private static class StubRecovery implements DeqpTestRunner.IRecovery { 77 /** 78 * {@inheritDoc} 79 */ 80 @Override setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider)81 public void setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider) { 82 } 83 84 /** 85 * {@inheritDoc} 86 */ 87 @Override setDevice(ITestDevice device)88 public void setDevice(ITestDevice device) { 89 } 90 91 /** 92 * {@inheritDoc} 93 */ 94 @Override onExecutionProgressed()95 public void onExecutionProgressed() { 96 } 97 98 /** 99 * {@inheritDoc} 100 */ 101 @Override recoverConnectionRefused()102 public void recoverConnectionRefused() throws DeviceNotAvailableException { 103 } 104 105 /** 106 * {@inheritDoc} 107 */ 108 @Override recoverComLinkKilled()109 public void recoverComLinkKilled() throws DeviceNotAvailableException { 110 } 111 }; 112 113 /** 114 * {@inheritDoc} 115 */ 116 @Override setUp()117 protected void setUp() throws Exception { 118 super.setUp(); 119 } 120 121 /** 122 * Test version of OpenGL ES. 123 */ testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion)124 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception { 125 final TestIdentifier testId = new TestIdentifier("dEQP-GLES" 126 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 127 + ".info", "version"); 128 129 final String testPath = "dEQP-GLES" 130 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 131 +".info.version"; 132 133 final String testTrie = "{dEQP-GLES" 134 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 135 + "{info{version}}}"; 136 137 final String resultCode = "Pass"; 138 139 /* MultiLineReceiver expects "\r\n" line ending. */ 140 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 141 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 142 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 143 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 144 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 145 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 146 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 147 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 148 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 149 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 150 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 151 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 152 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 153 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 154 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 155 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 156 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 157 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 158 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 159 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 160 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 161 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 162 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 163 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 164 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 165 + "INSTRUMENTATION_CODE: 0\r\n"; 166 167 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 168 ITestInvocationListener mockListener 169 = EasyMock.createStrictMock(ITestInvocationListener.class); 170 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 171 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 172 tests.add(testId); 173 174 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 175 instance.put(testId, DEFAULT_INSTANCE_ARGS); 176 177 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, 178 "dEQP-GLES" + Integer.toString(requiredMajorVersion) 179 + (requiredMinorVersion > 0 ? Integer.toString(requiredMinorVersion) : ""), 180 tests, instance); 181 deqpTest.setAbi(UnitTests.ABI); 182 183 int version = (majorVersion << 16) | minorVersion; 184 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 185 .andReturn(Integer.toString(version)).atLeastOnce(); 186 187 if (majorVersion > requiredMajorVersion 188 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) { 189 190 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 191 .andReturn("").once(); 192 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 193 EasyMock.eq(true), 194 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))) 195 .andReturn(null).once(); 196 197 expectRenderConfigQuery(mockDevice, requiredMajorVersion, 198 requiredMinorVersion); 199 200 String commandLine = String.format( 201 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 202 + "--deqp-screen-rotation=unspecified " 203 + "--deqp-surface-type=window " 204 + "--deqp-log-images=disable " 205 + "--deqp-watchdog=enable", 206 CASE_LIST_FILE_NAME); 207 208 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 209 output); 210 211 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 212 .andReturn("").once(); 213 } 214 215 mockListener.testRunStarted(ID, 1); 216 EasyMock.expectLastCall().once(); 217 218 mockListener.testStarted(EasyMock.eq(testId)); 219 EasyMock.expectLastCall().once(); 220 221 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 222 EasyMock.expectLastCall().once(); 223 224 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 225 EasyMock.expectLastCall().once(); 226 227 EasyMock.replay(mockDevice, mockIDevice); 228 EasyMock.replay(mockListener); 229 230 deqpTest.setDevice(mockDevice); 231 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 232 deqpTest.run(mockListener); 233 234 EasyMock.verify(mockListener); 235 EasyMock.verify(mockDevice, mockIDevice); 236 } 237 expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion)238 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, 239 int minorVersion) throws Exception { 240 expectRenderConfigQuery(mockDevice, 241 String.format("--deqp-gl-config-name=rgba8888d24s8 " 242 + "--deqp-screen-rotation=unspecified " 243 + "--deqp-surface-type=window " 244 + "--deqp-gl-major-version=%d " 245 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion)); 246 } 247 expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)248 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine) 249 throws Exception { 250 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); 251 } 252 expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, String output)253 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, 254 String output) throws Exception { 255 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" 256 + "INSTRUMENTATION_CODE: 0\r\n"; 257 final String command = String.format( 258 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " 259 + "\"%s\" %s", 260 AbiUtils.createAbiFlag(UnitTests.ABI.getName()), commandLine, 261 QUERY_INSTRUMENTATION_NAME); 262 263 mockDevice.executeShellCommand(EasyMock.eq(command), 264 EasyMock.<IShellOutputReceiver>notNull()); 265 266 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 267 @Override 268 public Object answer() { 269 IShellOutputReceiver receiver 270 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 271 272 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length()); 273 receiver.flush(); 274 275 return null; 276 } 277 }); 278 } 279 280 /** 281 * Test that result code produces correctly pass or fail. 282 */ testResultCode(final String resultCode, boolean pass)283 private void testResultCode(final String resultCode, boolean pass) throws Exception { 284 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version"); 285 final String testPath = "dEQP-GLES3.info.version"; 286 final String testTrie = "{dEQP-GLES3{info{version}}}"; 287 288 /* MultiLineReceiver expects "\r\n" line ending. */ 289 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 290 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 291 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 292 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 293 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 294 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 295 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 296 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 297 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 298 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 299 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 300 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 301 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 302 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 303 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 304 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 305 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 306 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 307 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 308 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 309 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 310 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 311 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 312 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 313 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 314 + "INSTRUMENTATION_CODE: 0\r\n"; 315 316 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 317 ITestInvocationListener mockListener 318 = EasyMock.createStrictMock(ITestInvocationListener.class); 319 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 320 321 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 322 tests.add(testId); 323 324 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 325 instance.put(testId, DEFAULT_INSTANCE_ARGS); 326 327 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 328 deqpTest.setAbi(UnitTests.ABI); 329 330 int version = 3 << 16; 331 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 332 .andReturn(Integer.toString(version)).atLeastOnce(); 333 334 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 335 .once(); 336 337 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 338 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))) 339 .andReturn(null).once(); 340 341 expectRenderConfigQuery(mockDevice, 3, 0); 342 343 String commandLine = String.format( 344 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 345 + "--deqp-screen-rotation=unspecified " 346 + "--deqp-surface-type=window " 347 + "--deqp-log-images=disable " 348 + "--deqp-watchdog=enable", 349 CASE_LIST_FILE_NAME); 350 351 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 352 353 mockListener.testRunStarted(ID, 1); 354 EasyMock.expectLastCall().once(); 355 356 mockListener.testStarted(EasyMock.eq(testId)); 357 EasyMock.expectLastCall().once(); 358 359 if (!pass) { 360 mockListener.testFailed(testId, 361 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n" 362 + resultCode + ": Detail" + resultCode); 363 364 EasyMock.expectLastCall().once(); 365 } 366 367 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 368 EasyMock.expectLastCall().once(); 369 370 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 371 EasyMock.expectLastCall().once(); 372 373 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 374 .once(); 375 376 EasyMock.replay(mockDevice, mockIDevice); 377 EasyMock.replay(mockListener); 378 379 deqpTest.setDevice(mockDevice); 380 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 381 deqpTest.run(mockListener); 382 383 EasyMock.verify(mockListener); 384 EasyMock.verify(mockDevice, mockIDevice); 385 } 386 387 /** 388 * Test running multiple test cases. 389 */ testRun_multipleTests()390 public void testRun_multipleTests() throws Exception { 391 /* MultiLineReceiver expects "\r\n" line ending. */ 392 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 393 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 394 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 395 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 396 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 397 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 398 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 399 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 400 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 401 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 402 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 403 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 404 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 405 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 406 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 407 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 408 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 409 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 410 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 411 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 412 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 413 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 414 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 415 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 416 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 417 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 418 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 419 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 420 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 421 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 422 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 423 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 424 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 425 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 426 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 427 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 428 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 429 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 430 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 431 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 432 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 433 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 434 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 435 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 436 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 437 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 438 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 439 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 440 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 441 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 442 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 443 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 444 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 445 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 446 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 447 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 448 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 449 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 450 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 451 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 452 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 453 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 454 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 455 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 456 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 457 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 458 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 459 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 460 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 461 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 462 + "INSTRUMENTATION_CODE: 0\r\n"; 463 464 final TestIdentifier[] testIds = { 465 new TestIdentifier("dEQP-GLES3.info", "vendor"), 466 new TestIdentifier("dEQP-GLES3.info", "renderer"), 467 new TestIdentifier("dEQP-GLES3.info", "version"), 468 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"), 469 new TestIdentifier("dEQP-GLES3.info", "extensions"), 470 new TestIdentifier("dEQP-GLES3.info", "render_target") 471 }; 472 473 final String[] testPaths = { 474 "dEQP-GLES3.info.vendor", 475 "dEQP-GLES3.info.renderer", 476 "dEQP-GLES3.info.version", 477 "dEQP-GLES3.info.shading_language_version", 478 "dEQP-GLES3.info.extensions", 479 "dEQP-GLES3.info.render_target" 480 }; 481 482 final String testTrie 483 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 484 485 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 486 ITestInvocationListener mockListener 487 = EasyMock.createStrictMock(ITestInvocationListener.class); 488 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 489 490 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 491 Map<TestIdentifier, List<Map<String, String>>> instances = new HashMap<>(); 492 493 for (TestIdentifier id : testIds) { 494 tests.add(id); 495 instances.put(id, DEFAULT_INSTANCE_ARGS); 496 } 497 498 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instances); 499 deqpTest.setAbi(UnitTests.ABI); 500 501 int version = 3 << 16; 502 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 503 .andReturn(Integer.toString(version)).atLeastOnce(); 504 505 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 506 .once(); 507 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 508 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))) 509 .andReturn(null).once(); 510 511 expectRenderConfigQuery(mockDevice, 3, 0); 512 513 String commandLine = String.format( 514 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 515 + "--deqp-screen-rotation=unspecified " 516 + "--deqp-surface-type=window " 517 + "--deqp-log-images=disable " 518 + "--deqp-watchdog=enable", 519 CASE_LIST_FILE_NAME); 520 521 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 522 523 mockListener.testRunStarted(ID, testPaths.length); 524 EasyMock.expectLastCall().once(); 525 526 for (int i = 0; i < testPaths.length; i++) { 527 mockListener.testStarted(EasyMock.eq(testIds[i])); 528 EasyMock.expectLastCall().once(); 529 530 mockListener.testEnded(EasyMock.eq(testIds[i]), 531 EasyMock.<Map<String, String>>notNull()); 532 533 EasyMock.expectLastCall().once(); 534 } 535 536 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 537 EasyMock.expectLastCall().once(); 538 539 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 540 .once(); 541 542 EasyMock.replay(mockDevice, mockIDevice); 543 EasyMock.replay(mockListener); 544 545 deqpTest.setDevice(mockDevice); 546 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 547 deqpTest.run(mockListener); 548 549 EasyMock.verify(mockListener); 550 EasyMock.verify(mockDevice, mockIDevice); 551 } 552 553 /** 554 * Test running a unexecutable test. 555 */ testRun_unexecutableTests()556 public void testRun_unexecutableTests() throws Exception { 557 final String instrumentationAnswerNoExecs = 558 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 559 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 560 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 561 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 562 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 563 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 564 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 565 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 566 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 567 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 568 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 569 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 570 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 571 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 572 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 573 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 574 + "INSTRUMENTATION_CODE: 0\r\n"; 575 576 final TestIdentifier[] testIds = { 577 new TestIdentifier("dEQP-GLES3.missing", "no"), 578 new TestIdentifier("dEQP-GLES3.missing", "nope"), 579 new TestIdentifier("dEQP-GLES3.missing", "donotwant"), 580 }; 581 582 final String[] testPaths = { 583 "dEQP-GLES3.missing.no", 584 "dEQP-GLES3.missing.nope", 585 "dEQP-GLES3.missing.donotwant", 586 }; 587 588 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 589 ITestInvocationListener mockListener 590 = EasyMock.createStrictMock(ITestInvocationListener.class); 591 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 592 593 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 594 Map<TestIdentifier, List<Map<String, String>>> instances = new HashMap<>(); 595 596 for (TestIdentifier id : testIds) { 597 tests.add(id); 598 instances.put(id, DEFAULT_INSTANCE_ARGS); 599 } 600 601 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instances); 602 deqpTest.setAbi(UnitTests.ABI); 603 604 int version = 3 << 16; 605 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 606 .andReturn(Integer.toString(version)).atLeastOnce(); 607 608 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 609 .once(); 610 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 611 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))) 612 .andReturn(null).once(); 613 614 expectRenderConfigQuery(mockDevice, 3, 0); 615 616 String commandLine = String.format( 617 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 618 + "--deqp-screen-rotation=unspecified " 619 + "--deqp-surface-type=window " 620 + "--deqp-log-images=disable " 621 + "--deqp-watchdog=enable", 622 CASE_LIST_FILE_NAME); 623 624 // first try 625 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 626 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 627 628 // splitting begins 629 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 630 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs); 631 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 632 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 633 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 634 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs); 635 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 636 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs); 637 638 mockListener.testRunStarted(ID, testPaths.length); 639 EasyMock.expectLastCall().once(); 640 641 for (int i = 0; i < testPaths.length; i++) { 642 mockListener.testStarted(EasyMock.eq(testIds[i])); 643 EasyMock.expectLastCall().once(); 644 645 mockListener.testFailed(EasyMock.eq(testIds[i]), 646 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n" 647 + "Abort: Test cannot be executed")); 648 EasyMock.expectLastCall().once(); 649 650 mockListener.testEnded(EasyMock.eq(testIds[i]), 651 EasyMock.<Map<String, String>>notNull()); 652 EasyMock.expectLastCall().once(); 653 } 654 655 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 656 EasyMock.expectLastCall().once(); 657 658 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") 659 .once(); 660 661 EasyMock.replay(mockDevice, mockIDevice); 662 EasyMock.replay(mockListener); 663 664 deqpTest.setDevice(mockDevice); 665 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 666 deqpTest.run(mockListener); 667 668 EasyMock.verify(mockListener); 669 EasyMock.verify(mockDevice, mockIDevice); 670 } 671 672 /** 673 * Test that test are left unexecuted if pm list query fails 674 */ testRun_queryPmListFailure()675 public void testRun_queryPmListFailure() 676 throws Exception { 677 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); 678 679 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 680 ITestInvocationListener mockListener 681 = EasyMock.createStrictMock(ITestInvocationListener.class); 682 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 683 tests.add(testId); 684 685 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 686 instance.put(testId, new ArrayList<Map<String,String>>(1)); 687 instance.get(testId).add(new HashMap<String,String>()); 688 instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8"); 689 instance.get(testId).iterator().next().put("rotation", "90"); 690 instance.get(testId).iterator().next().put("surfacetype", "window"); 691 692 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 693 deqpTest.setAbi(UnitTests.ABI); 694 deqpTest.setDevice(mockDevice); 695 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 696 697 int version = 3 << 16; 698 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 699 .andReturn(Integer.toString(version)).atLeastOnce(); 700 701 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 702 .andReturn("not a valid format"); 703 704 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 705 andReturn("").once(); 706 707 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 708 EasyMock.eq(true), 709 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 710 .once(); 711 712 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 713 .andReturn("").once(); 714 715 mockListener.testRunStarted(ID, 1); 716 EasyMock.expectLastCall().once(); 717 718 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 719 EasyMock.expectLastCall().once(); 720 721 EasyMock.replay(mockDevice); 722 EasyMock.replay(mockListener); 723 deqpTest.run(mockListener); 724 EasyMock.verify(mockListener); 725 EasyMock.verify(mockDevice); 726 } 727 728 /** 729 * Test that test are left unexecuted if renderablity query fails 730 */ testRun_queryRenderabilityFailure()731 public void testRun_queryRenderabilityFailure() 732 throws Exception { 733 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); 734 735 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 736 ITestInvocationListener mockListener 737 = EasyMock.createStrictMock(ITestInvocationListener.class); 738 739 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 740 tests.add(testId); 741 742 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 743 instance.put(testId, new ArrayList<Map<String,String>>(1)); 744 instance.get(testId).add(new HashMap<String,String>()); 745 instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8"); 746 instance.get(testId).iterator().next().put("rotation", "unspecified"); 747 instance.get(testId).iterator().next().put("surfacetype", "window"); 748 749 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 750 deqpTest.setAbi(UnitTests.ABI); 751 deqpTest.setDevice(mockDevice); 752 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 753 754 int version = 3 << 16; 755 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 756 .andReturn(Integer.toString(version)).atLeastOnce(); 757 758 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 759 andReturn("").once(); 760 761 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 762 EasyMock.eq(true), 763 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 764 .once(); 765 766 expectRenderConfigQueryAndReturn(mockDevice, 767 "--deqp-gl-config-name=rgba8888d24s8 " 768 + "--deqp-screen-rotation=unspecified " 769 + "--deqp-surface-type=window " 770 + "--deqp-gl-major-version=3 " 771 + "--deqp-gl-minor-version=0", "Maybe?"); 772 773 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 774 .andReturn("").once(); 775 776 mockListener.testRunStarted(ID, 1); 777 EasyMock.expectLastCall().once(); 778 779 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 780 EasyMock.expectLastCall().once(); 781 782 EasyMock.replay(mockDevice); 783 EasyMock.replay(mockListener); 784 deqpTest.run(mockListener); 785 EasyMock.verify(mockListener); 786 EasyMock.verify(mockDevice); 787 } 788 789 /** 790 * Test that orientation is supplied to runner correctly 791 */ testOrientation(final String rotation, final String featureString)792 private void testOrientation(final String rotation, final String featureString) 793 throws Exception { 794 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); 795 final String testPath = "dEQP-GLES3.orientation.test"; 796 final String testTrie = "{dEQP-GLES3{orientation{test}}}"; 797 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 798 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 799 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 800 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 801 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 802 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 803 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 804 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 805 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 806 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 807 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 808 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 809 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 810 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 811 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 812 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 813 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 814 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 815 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 816 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 817 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 818 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 819 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 820 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 821 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 822 + "INSTRUMENTATION_CODE: 0\r\n"; 823 824 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 825 ITestInvocationListener mockListener 826 = EasyMock.createStrictMock(ITestInvocationListener.class); 827 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 828 829 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 830 tests.add(testId); 831 832 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 833 instance.put(testId, new ArrayList<Map<String,String>>(1)); 834 instance.get(testId).add(new HashMap<String,String>()); 835 instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8"); 836 instance.get(testId).iterator().next().put("rotation", rotation); 837 instance.get(testId).iterator().next().put("surfacetype", "window"); 838 839 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 840 deqpTest.setAbi(UnitTests.ABI); 841 deqpTest.setDevice(mockDevice); 842 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 843 844 int version = 3 << 16; 845 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 846 .andReturn(Integer.toString(version)).atLeastOnce(); 847 848 if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) { 849 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 850 .andReturn(featureString); 851 } 852 853 final boolean isPortraitOrientation = 854 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) || 855 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); 856 final boolean isLandscapeOrientation = 857 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) || 858 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); 859 final boolean executable = 860 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) || 861 (isPortraitOrientation && 862 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || 863 (isLandscapeOrientation && 864 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); 865 866 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 867 andReturn("").once(); 868 869 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 870 EasyMock.eq(true), 871 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 872 .once(); 873 874 if (executable) { 875 expectRenderConfigQuery(mockDevice, String.format( 876 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " 877 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 878 + "--deqp-gl-minor-version=0", rotation)); 879 880 String commandLine = String.format( 881 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 882 + "--deqp-screen-rotation=%s " 883 + "--deqp-surface-type=window " 884 + "--deqp-log-images=disable " 885 + "--deqp-watchdog=enable", 886 CASE_LIST_FILE_NAME, rotation); 887 888 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 889 output); 890 } 891 892 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 893 .andReturn("").once(); 894 895 mockListener.testRunStarted(ID, 1); 896 EasyMock.expectLastCall().once(); 897 898 mockListener.testStarted(EasyMock.eq(testId)); 899 EasyMock.expectLastCall().once(); 900 901 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 902 EasyMock.expectLastCall().once(); 903 904 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 905 EasyMock.expectLastCall().once(); 906 907 EasyMock.replay(mockDevice, mockIDevice); 908 EasyMock.replay(mockListener); 909 deqpTest.run(mockListener); 910 EasyMock.verify(mockListener); 911 EasyMock.verify(mockDevice, mockIDevice); 912 } 913 914 /** 915 * Test OpeGL ES3 tests on device with OpenGL ES2. 916 */ testRun_require30DeviceVersion20()917 public void testRun_require30DeviceVersion20() throws Exception { 918 testGlesVersion(3, 0, 2, 0); 919 } 920 921 /** 922 * Test OpeGL ES3.1 tests on device with OpenGL ES2. 923 */ testRun_require31DeviceVersion20()924 public void testRun_require31DeviceVersion20() throws Exception { 925 testGlesVersion(3, 1, 2, 0); 926 } 927 928 /** 929 * Test OpeGL ES3 tests on device with OpenGL ES3. 930 */ testRun_require30DeviceVersion30()931 public void testRun_require30DeviceVersion30() throws Exception { 932 testGlesVersion(3, 0, 3, 0); 933 } 934 935 /** 936 * Test OpeGL ES3.1 tests on device with OpenGL ES3. 937 */ testRun_require31DeviceVersion30()938 public void testRun_require31DeviceVersion30() throws Exception { 939 testGlesVersion(3, 1, 3, 0); 940 } 941 942 /** 943 * Test OpeGL ES3 tests on device with OpenGL ES3.1. 944 */ testRun_require30DeviceVersion31()945 public void testRun_require30DeviceVersion31() throws Exception { 946 testGlesVersion(3, 0, 3, 1); 947 } 948 949 /** 950 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. 951 */ testRun_require31DeviceVersion31()952 public void testRun_require31DeviceVersion31() throws Exception { 953 testGlesVersion(3, 1, 3, 1); 954 } 955 956 /** 957 * Test dEQP Pass result code. 958 */ testRun_resultPass()959 public void testRun_resultPass() throws Exception { 960 testResultCode("Pass", true); 961 } 962 963 /** 964 * Test dEQP Fail result code. 965 */ testRun_resultFail()966 public void testRun_resultFail() throws Exception { 967 testResultCode("Fail", false); 968 } 969 970 /** 971 * Test dEQP NotSupported result code. 972 */ testRun_resultNotSupported()973 public void testRun_resultNotSupported() throws Exception { 974 testResultCode("NotSupported", true); 975 } 976 977 /** 978 * Test dEQP QualityWarning result code. 979 */ testRun_resultQualityWarning()980 public void testRun_resultQualityWarning() throws Exception { 981 testResultCode("QualityWarning", true); 982 } 983 984 /** 985 * Test dEQP CompatibilityWarning result code. 986 */ testRun_resultCompatibilityWarning()987 public void testRun_resultCompatibilityWarning() throws Exception { 988 testResultCode("CompatibilityWarning", true); 989 } 990 991 /** 992 * Test dEQP ResourceError result code. 993 */ testRun_resultResourceError()994 public void testRun_resultResourceError() throws Exception { 995 testResultCode("ResourceError", false); 996 } 997 998 /** 999 * Test dEQP InternalError result code. 1000 */ testRun_resultInternalError()1001 public void testRun_resultInternalError() throws Exception { 1002 testResultCode("InternalError", false); 1003 } 1004 1005 /** 1006 * Test dEQP Crash result code. 1007 */ testRun_resultCrash()1008 public void testRun_resultCrash() throws Exception { 1009 testResultCode("Crash", false); 1010 } 1011 1012 /** 1013 * Test dEQP Timeout result code. 1014 */ testRun_resultTimeout()1015 public void testRun_resultTimeout() throws Exception { 1016 testResultCode("Timeout", false); 1017 } 1018 /** 1019 * Test dEQP Orientation 1020 */ testRun_orientationLandscape()1021 public void testRun_orientationLandscape() throws Exception { 1022 testOrientation("90", ALL_FEATURES); 1023 } 1024 1025 /** 1026 * Test dEQP Orientation 1027 */ testRun_orientationPortrait()1028 public void testRun_orientationPortrait() throws Exception { 1029 testOrientation("0", ALL_FEATURES); 1030 } 1031 1032 /** 1033 * Test dEQP Orientation 1034 */ testRun_orientationReverseLandscape()1035 public void testRun_orientationReverseLandscape() throws Exception { 1036 testOrientation("270", ALL_FEATURES); 1037 } 1038 1039 /** 1040 * Test dEQP Orientation 1041 */ testRun_orientationReversePortrait()1042 public void testRun_orientationReversePortrait() throws Exception { 1043 testOrientation("180", ALL_FEATURES); 1044 } 1045 1046 /** 1047 * Test dEQP Orientation 1048 */ testRun_orientationUnspecified()1049 public void testRun_orientationUnspecified() throws Exception { 1050 testOrientation("unspecified", ALL_FEATURES); 1051 } 1052 1053 /** 1054 * Test dEQP Orientation with limited features 1055 */ testRun_orientationUnspecifiedLimitedFeatures()1056 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception { 1057 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); 1058 } 1059 1060 /** 1061 * Test dEQP Orientation with limited features 1062 */ testRun_orientationLandscapeLimitedFeatures()1063 public void testRun_orientationLandscapeLimitedFeatures() throws Exception { 1064 testOrientation("90", ONLY_LANDSCAPE_FEATURES); 1065 } 1066 1067 /** 1068 * Test dEQP Orientation with limited features 1069 */ testRun_orientationPortraitLimitedFeatures()1070 public void testRun_orientationPortraitLimitedFeatures() throws Exception { 1071 testOrientation("0", ONLY_LANDSCAPE_FEATURES); 1072 } 1073 1074 /** 1075 * Test dEQP unsupported pixel format 1076 */ testRun_unsupportedPixelFormat()1077 public void testRun_unsupportedPixelFormat() throws Exception { 1078 final String pixelFormat = "rgba5658d16m4"; 1079 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test"); 1080 final String testPath = "dEQP-GLES3.pixelformat.test"; 1081 final String testTrie = "{dEQP-GLES3{pixelformat{test}}}"; 1082 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1083 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1084 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1085 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1086 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1087 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1088 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1089 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1090 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1091 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1092 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1093 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1094 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1095 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1096 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1097 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1098 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1099 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1100 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1101 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1102 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1103 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1104 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1105 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1106 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1107 + "INSTRUMENTATION_CODE: 0\r\n"; 1108 1109 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1110 ITestInvocationListener mockListener 1111 = EasyMock.createStrictMock(ITestInvocationListener.class); 1112 1113 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1114 tests.add(testId); 1115 1116 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 1117 instance.put(testId, new ArrayList<Map<String,String>>(1)); 1118 instance.get(testId).add(new HashMap<String,String>()); 1119 instance.get(testId).iterator().next().put("glconfig", pixelFormat); 1120 instance.get(testId).iterator().next().put("rotation", "unspecified"); 1121 instance.get(testId).iterator().next().put("surfacetype", "window"); 1122 1123 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 1124 deqpTest.setAbi(UnitTests.ABI); 1125 deqpTest.setDevice(mockDevice); 1126 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 1127 1128 int version = 3 << 16; 1129 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1130 .andReturn(Integer.toString(version)).atLeastOnce(); 1131 1132 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1133 andReturn("").once(); 1134 1135 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1136 EasyMock.eq(true), 1137 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 1138 .once(); 1139 1140 expectRenderConfigQueryAndReturn(mockDevice, String.format( 1141 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " 1142 + "--deqp-surface-type=window " 1143 + "--deqp-gl-major-version=3 " 1144 + "--deqp-gl-minor-version=0", pixelFormat), "No"); 1145 1146 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 1147 .andReturn("").once(); 1148 1149 mockListener.testRunStarted(ID, 1); 1150 EasyMock.expectLastCall().once(); 1151 1152 mockListener.testStarted(EasyMock.eq(testId)); 1153 EasyMock.expectLastCall().once(); 1154 1155 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 1156 EasyMock.expectLastCall().once(); 1157 1158 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 1159 EasyMock.expectLastCall().once(); 1160 1161 EasyMock.replay(mockDevice); 1162 EasyMock.replay(mockListener); 1163 deqpTest.run(mockListener); 1164 EasyMock.verify(mockListener); 1165 EasyMock.verify(mockDevice); 1166 } 1167 1168 /** 1169 * Test dEQP with multiple instances 1170 */ testRun_multipleInstances()1171 public void testRun_multipleInstances() throws Exception { 1172 final String instrumentationAnswerConfigAPass1 = 1173 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1174 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1175 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1176 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1177 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1178 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1179 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1180 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1181 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1182 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1183 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1184 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1185 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1186 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1187 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1188 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.passall\r\n" 1189 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1190 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1191 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1192 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1193 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1194 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1195 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1196 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1197 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.failone\r\n" 1198 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1199 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1200 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1201 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1202 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1203 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1204 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1205 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1206 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n" 1207 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; // early eof 1208 final String instrumentationAnswerConfigAPass2 = 1209 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1210 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1211 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1212 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1213 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1214 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1215 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1216 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1217 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1218 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1219 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1220 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1221 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1222 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1223 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1224 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n" 1225 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; // early eof 1226 final String instrumentationAnswerConfigBPass1 = 1227 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1228 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1229 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1230 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1231 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1232 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1233 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1234 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1235 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1236 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1237 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1238 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1239 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1240 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1241 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1242 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.passall\r\n" 1243 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1244 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1245 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1246 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1247 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1248 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1249 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1250 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1251 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.skipone\r\n" 1252 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1253 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1254 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1255 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1256 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1257 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1258 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1259 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1260 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1261 + "INSTRUMENTATION_CODE: 0\r\n"; 1262 final String instrumentationAnswerConfigBPass2 = 1263 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1264 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1265 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1266 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1267 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1268 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1269 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1270 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1271 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1272 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1273 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1274 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1275 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1276 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1277 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1278 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n" 1279 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1280 + "INSTRUMENTATION_STATUS: dEQP-TerminateTestCase-Reason=Magic\r\n" 1281 + "INSTRUMENTATION_STATUS: dEQP-EventType=TerminateTestCase\r\n" 1282 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1283 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1284 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1285 + "INSTRUMENTATION_CODE: 0\r\n"; 1286 final String instrumentationAnswerConfigCPass1 = 1287 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1288 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1289 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1290 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1291 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1292 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1293 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1294 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1295 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1296 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1297 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1298 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1299 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1300 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1301 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1302 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.failone\r\n" 1303 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1304 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 1305 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 1306 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1307 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1308 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1309 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1310 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1311 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1312 + "INSTRUMENTATION_CODE: 0\r\n"; 1313 final String instrumentationAnswerConfigCPass2 = 1314 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1315 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1316 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1317 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1318 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1319 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1320 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1321 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1322 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1323 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1324 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1325 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1326 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1327 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1328 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1329 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n" 1330 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1331 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1332 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1333 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1334 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1335 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1336 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1337 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1338 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1339 + "INSTRUMENTATION_CODE: 0\r\n"; 1340 1341 final TestIdentifier[] testIds = { 1342 new TestIdentifier("dEQP-GLES3.instances", "passall"), 1343 new TestIdentifier("dEQP-GLES3.instances", "failone"), 1344 new TestIdentifier("dEQP-GLES3.instances", "crashtwo"), 1345 new TestIdentifier("dEQP-GLES3.instances", "skipone"), 1346 }; 1347 1348 final String[] testPaths = { 1349 "dEQP-GLES3.instances.passall", 1350 "dEQP-GLES3.instances.failone", 1351 "dEQP-GLES3.instances.crashtwo", 1352 "dEQP-GLES3.instances.skipone", 1353 }; 1354 1355 Map<String,String> supportedConfigA = new HashMap<>(); 1356 supportedConfigA.put("glconfig", "rgba8888d24s8"); 1357 supportedConfigA.put("rotation", "unspecified"); 1358 supportedConfigA.put("surfacetype", "window"); 1359 1360 Map<String,String> supportedConfigB = new HashMap<>(); 1361 supportedConfigB.put("glconfig", "rgba8888d24s8"); 1362 supportedConfigB.put("rotation", "90"); 1363 supportedConfigB.put("surfacetype", "window"); 1364 1365 Map<String,String> supportedConfigC = new HashMap<>(); 1366 supportedConfigC.put("glconfig", "rgba8888d24s8"); 1367 supportedConfigC.put("rotation", "180"); 1368 supportedConfigC.put("surfacetype", "window"); 1369 1370 Map<String,String> unsupportedConfig = new HashMap<>(); 1371 unsupportedConfig.put("glconfig", "rgb565d16s0"); 1372 unsupportedConfig.put("rotation", "unspecified"); 1373 unsupportedConfig.put("surfacetype", "window"); 1374 1375 Map<TestIdentifier, List<Map<String, String>>> instances = new HashMap<>(); 1376 1377 // pass all 1378 instances.put(testIds[0], new ArrayList<Map<String,String>>()); 1379 instances.get(testIds[0]).add(supportedConfigA); 1380 instances.get(testIds[0]).add(supportedConfigB); 1381 1382 // fail one 1383 instances.put(testIds[1], new ArrayList<Map<String,String>>()); 1384 instances.get(testIds[1]).add(supportedConfigA); 1385 instances.get(testIds[1]).add(supportedConfigC); 1386 1387 // crash two 1388 instances.put(testIds[2], new ArrayList<Map<String,String>>()); 1389 instances.get(testIds[2]).add(supportedConfigA); 1390 instances.get(testIds[2]).add(supportedConfigC); 1391 instances.get(testIds[2]).add(supportedConfigB); 1392 1393 // skip one 1394 instances.put(testIds[3], new ArrayList<Map<String,String>>()); 1395 instances.get(testIds[3]).add(supportedConfigB); 1396 instances.get(testIds[3]).add(unsupportedConfig); 1397 1398 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1399 for (TestIdentifier id : testIds) { 1400 tests.add(id); 1401 } 1402 1403 ITestInvocationListener mockListener 1404 = EasyMock.createStrictMock(ITestInvocationListener.class); 1405 IMocksControl orderedControl = EasyMock.createStrictControl(); 1406 ITestDevice mockDevice = orderedControl.createMock(ITestDevice.class); 1407 IDevice mockIDevice = orderedControl.createMock(IDevice.class); 1408 1409 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instances); 1410 deqpTest.setAbi(UnitTests.ABI); 1411 deqpTest.setDevice(mockDevice); 1412 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 1413 1414 int version = 3 << 16; 1415 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1416 .andReturn(Integer.toString(version)).atLeastOnce(); 1417 1418 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1419 andReturn("").once(); 1420 1421 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1422 EasyMock.eq(true), 1423 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 1424 .once(); 1425 1426 // query config A 1427 expectRenderConfigQueryAndReturn(mockDevice, 1428 "--deqp-gl-config-name=rgba8888d24s8 " 1429 + "--deqp-screen-rotation=unspecified " 1430 + "--deqp-surface-type=window " 1431 + "--deqp-gl-major-version=3 " 1432 + "--deqp-gl-minor-version=0", "Yes"); 1433 1434 // run config A - first pass 1435 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1436 "{dEQP-GLES3{instances{passall,failone,crashtwo}}}", 1437 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1438 + " --deqp-gl-config-name=rgba8888d24s8 " 1439 + "--deqp-screen-rotation=unspecified " 1440 + "--deqp-surface-type=window " 1441 + "--deqp-log-images=disable " 1442 + "--deqp-watchdog=enable", instrumentationAnswerConfigAPass1); 1443 1444 // run config A - second pass 1445 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1446 "{dEQP-GLES3{instances{crashtwo}}}", 1447 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1448 + " --deqp-gl-config-name=rgba8888d24s8 " 1449 + "--deqp-screen-rotation=unspecified " 1450 + "--deqp-surface-type=window " 1451 + "--deqp-log-images=disable " 1452 + "--deqp-watchdog=enable", instrumentationAnswerConfigAPass2); 1453 1454 // query for config B 1455 1456 EasyMock.expect(mockDevice.executeShellCommand("pm list features")).andReturn(ALL_FEATURES) 1457 .once(); 1458 1459 expectRenderConfigQueryAndReturn(mockDevice, 1460 "--deqp-gl-config-name=rgba8888d24s8 " 1461 + "--deqp-screen-rotation=90 " 1462 + "--deqp-surface-type=window " 1463 + "--deqp-gl-major-version=3 " 1464 + "--deqp-gl-minor-version=0", "Yes"); 1465 1466 // run for config B - first pass 1467 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1468 "{dEQP-GLES3{instances{passall,skipone}}}", 1469 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1470 + " --deqp-gl-config-name=rgba8888d24s8 " 1471 + "--deqp-screen-rotation=90 " 1472 + "--deqp-surface-type=window " 1473 + "--deqp-log-images=disable " 1474 + "--deqp-watchdog=enable", instrumentationAnswerConfigBPass1); 1475 1476 // query for config C 1477 expectRenderConfigQueryAndReturn(mockDevice, 1478 "--deqp-gl-config-name=rgba8888d24s8 " 1479 + "--deqp-screen-rotation=180 " 1480 + "--deqp-surface-type=window " 1481 + "--deqp-gl-major-version=3 " 1482 + "--deqp-gl-minor-version=0", "Yes"); 1483 1484 // run for config C - first pass 1485 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1486 "{dEQP-GLES3{instances{failone}}}", 1487 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1488 + " --deqp-gl-config-name=rgba8888d24s8 " 1489 + "--deqp-screen-rotation=180 " 1490 + "--deqp-surface-type=window " 1491 + "--deqp-log-images=disable " 1492 + "--deqp-watchdog=enable", instrumentationAnswerConfigCPass1); 1493 1494 // run for config C - second pass 1495 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1496 "{dEQP-GLES3{instances{crashtwo}}}", 1497 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1498 + " --deqp-gl-config-name=rgba8888d24s8 " 1499 + "--deqp-screen-rotation=180 " 1500 + "--deqp-surface-type=window " 1501 + "--deqp-log-images=disable " 1502 + "--deqp-watchdog=enable", instrumentationAnswerConfigCPass2); 1503 1504 // run for config B - second pass (crashtwo has been deferred due to its instability) 1505 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1506 "{dEQP-GLES3{instances{crashtwo}}}", 1507 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1508 + " --deqp-gl-config-name=rgba8888d24s8 " 1509 + "--deqp-screen-rotation=90 " 1510 + "--deqp-surface-type=window " 1511 + "--deqp-log-images=disable " 1512 + "--deqp-watchdog=enable", instrumentationAnswerConfigBPass2); 1513 1514 // query for unsupported config 1515 expectRenderConfigQueryAndReturn(mockDevice, 1516 "--deqp-gl-config-name=rgb565d16s0 " 1517 + "--deqp-screen-rotation=unspecified " 1518 + "--deqp-surface-type=window " 1519 + "--deqp-gl-major-version=3 " 1520 + "--deqp-gl-minor-version=0", "No"); 1521 1522 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) 1523 .andReturn("").once(); 1524 1525 mockListener.testRunStarted(ID, 4); 1526 EasyMock.expectLastCall().once(); 1527 1528 // pass all 1529 mockListener.testStarted(EasyMock.eq(testIds[0])); 1530 EasyMock.expectLastCall().once(); 1531 1532 mockListener.testEnded(EasyMock.eq(testIds[0]), EasyMock.<Map<String, String>>notNull()); 1533 EasyMock.expectLastCall().once(); 1534 1535 // fail one 1536 mockListener.testStarted(EasyMock.eq(testIds[1])); 1537 EasyMock.expectLastCall().once(); 1538 1539 mockListener.testFailed(testIds[1], 1540 "=== with config {glformat=rgba8888d24s8,rotation=180,surfacetype=window} ===\n" 1541 + "Fail: Fail"); 1542 EasyMock.expectLastCall().once(); 1543 1544 mockListener.testEnded(EasyMock.eq(testIds[1]), EasyMock.<Map<String, String>>notNull()); 1545 EasyMock.expectLastCall().once(); 1546 1547 // crash two 1548 mockListener.testStarted(EasyMock.eq(testIds[2])); 1549 EasyMock.expectLastCall().once(); 1550 1551 mockListener.testFailed(testIds[2], 1552 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n" 1553 + "Crash: Incomplete test log\n" 1554 + "=== with config {glformat=rgba8888d24s8,rotation=90,surfacetype=window} ===\n" 1555 + "Terminated: Magic"); 1556 EasyMock.expectLastCall().once(); 1557 1558 mockListener.testEnded(EasyMock.eq(testIds[2]), EasyMock.<Map<String, String>>notNull()); 1559 EasyMock.expectLastCall().once(); 1560 1561 // skip one 1562 mockListener.testStarted(EasyMock.eq(testIds[3])); 1563 EasyMock.expectLastCall().once(); 1564 1565 mockListener.testEnded(EasyMock.eq(testIds[3]), EasyMock.<Map<String, String>>notNull()); 1566 EasyMock.expectLastCall().once(); 1567 1568 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 1569 EasyMock.expectLastCall().once(); 1570 1571 orderedControl.replay(); 1572 EasyMock.replay(mockListener); 1573 deqpTest.setRecovery(new StubRecovery()); 1574 deqpTest.run(mockListener); 1575 EasyMock.verify(mockListener); 1576 orderedControl.verify(); 1577 } 1578 testMultipleInstancesLossOfDeviceMidInstance(final boolean recoverySuccessful)1579 private void testMultipleInstancesLossOfDeviceMidInstance(final boolean recoverySuccessful) 1580 throws Exception { 1581 final String instrumentationAnswerFine = 1582 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1583 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1584 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1585 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1586 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1587 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1588 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1589 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1590 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1591 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1592 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1593 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1594 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1595 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1596 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1597 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.loss.instance\r\n" 1598 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1599 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1600 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1601 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1602 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1603 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1604 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1605 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1606 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1607 + "INSTRUMENTATION_CODE: 0\r\n"; 1608 final String instrumentationAnswerCrash = 1609 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1610 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1611 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1612 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1613 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1614 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1615 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1616 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1617 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1618 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1619 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1620 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1621 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1622 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1623 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1624 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.loss.instance\r\n" 1625 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; // early <EOF> 1626 1627 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.loss", "instance"); 1628 1629 Map<String,String> supportedConfigA = new HashMap<>(); 1630 supportedConfigA.put("glconfig", "rgba8888d24s8"); 1631 supportedConfigA.put("rotation", "unspecified"); 1632 supportedConfigA.put("surfacetype", "window"); 1633 1634 Map<String,String> supportedConfigB = new HashMap<>(); 1635 supportedConfigB.put("glconfig", "rgba8888d24s8"); 1636 supportedConfigB.put("rotation", "90"); 1637 supportedConfigB.put("surfacetype", "window"); 1638 1639 Map<String,String> supportedConfigC = new HashMap<>(); 1640 supportedConfigC.put("glconfig", "rgba8888d24s8"); 1641 supportedConfigC.put("rotation", "180"); 1642 supportedConfigC.put("surfacetype", "window"); 1643 1644 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 1645 tests.add(testId); 1646 1647 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 1648 instance.put(testId, new ArrayList<Map<String,String>>()); 1649 instance.get(testId).add(supportedConfigA); 1650 instance.get(testId).add(supportedConfigB); 1651 instance.get(testId).add(supportedConfigC); 1652 1653 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1654 ITestInvocationListener mockListener 1655 = EasyMock.createStrictMock(ITestInvocationListener.class); 1656 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1657 DeqpTestRunner.IRecovery mockRecovery = EasyMock.createMock(DeqpTestRunner.IRecovery.class); 1658 1659 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 1660 deqpTest.setAbi(UnitTests.ABI); 1661 deqpTest.setDevice(mockDevice); 1662 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 1663 deqpTest.setRecovery(mockRecovery); 1664 1665 int version = 3 << 16; 1666 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1667 .andReturn(Integer.toString(version)).atLeastOnce(); 1668 EasyMock.expect(mockDevice.executeShellCommand("pm list features")).andReturn(ALL_FEATURES) 1669 .anyTimes(); 1670 1671 mockRecovery.onExecutionProgressed(); 1672 EasyMock.expectLastCall().atLeastOnce(); 1673 1674 mockRecovery.setDevice(mockDevice); 1675 EasyMock.expectLastCall().atLeastOnce(); 1676 1677 if (!recoverySuccessful) { 1678 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1679 andReturn("").once(); 1680 } else { 1681 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 1682 andReturn("").times(2); 1683 } 1684 1685 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 1686 EasyMock.eq(true), 1687 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 1688 .once(); 1689 1690 // query config A 1691 expectRenderConfigQueryAndReturn(mockDevice, 1692 "--deqp-gl-config-name=rgba8888d24s8 " 1693 + "--deqp-screen-rotation=unspecified " 1694 + "--deqp-surface-type=window " 1695 + "--deqp-gl-major-version=3 " 1696 + "--deqp-gl-minor-version=0", "Yes"); 1697 1698 // run config A 1699 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1700 "{dEQP-GLES3{loss{instance}}}", 1701 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1702 + " --deqp-gl-config-name=rgba8888d24s8 " 1703 + "--deqp-screen-rotation=unspecified " 1704 + "--deqp-surface-type=window " 1705 + "--deqp-log-images=disable " 1706 + "--deqp-watchdog=enable", instrumentationAnswerFine); 1707 1708 // query config B 1709 expectRenderConfigQueryAndReturn(mockDevice, 1710 "--deqp-gl-config-name=rgba8888d24s8 " 1711 + "--deqp-screen-rotation=90 " 1712 + "--deqp-surface-type=window " 1713 + "--deqp-gl-major-version=3 " 1714 + "--deqp-gl-minor-version=0", "Yes"); 1715 1716 // run config B 1717 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) 1718 .andReturn("").once(); 1719 1720 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) 1721 .andReturn("").once(); 1722 1723 EasyMock.expect(mockDevice.pushString("{dEQP-GLES3{loss{instance}}}\n", CASE_LIST_FILE_NAME)) 1724 .andReturn(true).once(); 1725 1726 String command = String.format( 1727 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"" 1728 + "--deqp-caselist-file=%s" 1729 + " --deqp-gl-config-name=rgba8888d24s8 " 1730 + "--deqp-screen-rotation=90 " 1731 + "--deqp-surface-type=window " 1732 + "--deqp-log-images=disable " 1733 + "--deqp-watchdog=enable\" " 1734 + "-e deqpLogData \"%s\" %s", 1735 AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, 1736 CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME); 1737 1738 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 1739 mockIDevice.executeShellCommand(EasyMock.eq(command), 1740 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 1741 EasyMock.isA(TimeUnit.class)); 1742 1743 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 1744 @Override 1745 public Object answer() throws ShellCommandUnresponsiveException { 1746 IShellOutputReceiver receiver 1747 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 1748 1749 receiver.addOutput(instrumentationAnswerCrash.getBytes(), 0, 1750 instrumentationAnswerCrash.length()); 1751 throw new ShellCommandUnresponsiveException(); 1752 } 1753 }); 1754 1755 if (!recoverySuccessful) { 1756 mockRecovery.recoverComLinkKilled(); 1757 EasyMock.expectLastCall().andThrow(new DeviceNotAvailableException()).once(); 1758 } else { 1759 mockRecovery.recoverComLinkKilled(); 1760 EasyMock.expectLastCall().once(); 1761 1762 // retry running config B 1763 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1764 "{dEQP-GLES3{loss{instance}}}", 1765 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1766 + " --deqp-gl-config-name=rgba8888d24s8 " 1767 + "--deqp-screen-rotation=90 " 1768 + "--deqp-surface-type=window " 1769 + "--deqp-log-images=disable " 1770 + "--deqp-watchdog=enable", instrumentationAnswerFine); 1771 1772 // query config C 1773 expectRenderConfigQueryAndReturn(mockDevice, 1774 "--deqp-gl-config-name=rgba8888d24s8 " 1775 + "--deqp-screen-rotation=180 " 1776 + "--deqp-surface-type=window " 1777 + "--deqp-gl-major-version=3 " 1778 + "--deqp-gl-minor-version=0", "Yes"); 1779 1780 // run config C 1781 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 1782 "{dEQP-GLES3{loss{instance}}}", 1783 "--deqp-caselist-file=" + CASE_LIST_FILE_NAME 1784 + " --deqp-gl-config-name=rgba8888d24s8 " 1785 + "--deqp-screen-rotation=180 " 1786 + "--deqp-surface-type=window " 1787 + "--deqp-log-images=disable " 1788 + "--deqp-watchdog=enable", instrumentationAnswerFine); 1789 } 1790 1791 mockListener.testRunStarted(ID, 1); 1792 EasyMock.expectLastCall().once(); 1793 1794 // result is reported only if device is available 1795 if (recoverySuccessful) { 1796 mockListener.testStarted(EasyMock.eq(testId)); 1797 EasyMock.expectLastCall().once(); 1798 1799 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); 1800 EasyMock.expectLastCall().once(); 1801 1802 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); 1803 EasyMock.expectLastCall().once(); 1804 } 1805 1806 EasyMock.replay(mockDevice, mockIDevice); 1807 EasyMock.replay(mockListener); 1808 EasyMock.replay(mockRecovery); 1809 1810 try { 1811 deqpTest.run(mockListener); 1812 1813 if (!recoverySuccessful) { 1814 fail("did not get DeviceNotAvailableException"); 1815 } 1816 } catch (DeviceNotAvailableException ex) { 1817 if (recoverySuccessful) { 1818 fail("got DeviceNotAvailableException"); 1819 } 1820 } 1821 1822 EasyMock.verify(mockRecovery); 1823 EasyMock.verify(mockListener); 1824 EasyMock.verify(mockDevice, mockIDevice); 1825 } 1826 1827 /** 1828 * Test dEQP with runner if device is lost during one of multiple instances. 1829 */ testRun_multipleInstancesLossOfDeviceMidInstance()1830 public void testRun_multipleInstancesLossOfDeviceMidInstance() throws Exception { 1831 testMultipleInstancesLossOfDeviceMidInstance(false); 1832 } 1833 1834 /** 1835 * Test dEQP with runner if device is lost during one of multiple instances but recovery 1836 * is successful. 1837 */ testRun_multipleInstancesLossOfDeviceMidInstanceAndRecovery()1838 public void testRun_multipleInstancesLossOfDeviceMidInstanceAndRecovery() throws Exception { 1839 testMultipleInstancesLossOfDeviceMidInstance(true); 1840 } 1841 1842 public static interface RecoverableTestDevice extends ITestDevice { recoverDevice()1843 public void recoverDevice() throws DeviceNotAvailableException; 1844 } 1845 1846 private static enum RecoveryEvent { 1847 PROGRESS, 1848 FAIL_CONNECTION_REFUSED, 1849 FAIL_LINK_KILLED, 1850 }; 1851 runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)1852 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events) 1853 throws DeviceNotAvailableException { 1854 for (RecoveryEvent event : events) { 1855 switch (event) { 1856 case PROGRESS: 1857 recovery.onExecutionProgressed(); 1858 break; 1859 case FAIL_CONNECTION_REFUSED: 1860 recovery.recoverConnectionRefused(); 1861 break; 1862 case FAIL_LINK_KILLED: 1863 recovery.recoverComLinkKilled(); 1864 break; 1865 } 1866 } 1867 } 1868 setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider)1869 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) { 1870 mockSleepProvider.sleep(EasyMock.gt(0)); 1871 EasyMock.expectLastCall().once(); 1872 } 1873 setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider)1874 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, 1875 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { 1876 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1877 andReturn("root 1234 com.drawelement.deqp").once(); 1878 1879 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1880 andReturn("").once(); 1881 1882 // Recovery checks if kill failed 1883 mockSleepProvider.sleep(EasyMock.gt(0)); 1884 EasyMock.expectLastCall().once(); 1885 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1886 andReturn("").once(); 1887 } 1888 setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)1889 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 1890 throws DeviceNotAvailableException { 1891 mockDevice.recoverDevice(); 1892 EasyMock.expectLastCall().once(); 1893 } 1894 setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)1895 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 1896 throws DeviceNotAvailableException { 1897 mockDevice.reboot(); 1898 EasyMock.expectLastCall().once(); 1899 } 1900 setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1901 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, 1902 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1903 throws DeviceNotAvailableException { 1904 switch (numConsecutiveErrors) { 1905 case 0: 1906 case 1: 1907 setRecoveryExpectationRecovery(mockDevice); 1908 return 2; 1909 case 2: 1910 setRecoveryExpectationReboot(mockDevice); 1911 return 3; 1912 default: 1913 return 4; 1914 } 1915 } 1916 setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)1917 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, 1918 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1919 throws DeviceNotAvailableException { 1920 switch (numConsecutiveErrors) { 1921 case 0: 1922 setRecoveryExpectationWait(mockSleepProvider); 1923 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1924 return 1; 1925 case 1: 1926 setRecoveryExpectationRecovery(mockDevice); 1927 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1928 return 2; 1929 case 2: 1930 setRecoveryExpectationReboot(mockDevice); 1931 return 3; 1932 default: 1933 return 4; 1934 } 1935 } 1936 setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)1937 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, 1938 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 1939 throws DeviceNotAvailableException { 1940 int numConsecutiveErrors = 0; 1941 for (RecoveryEvent event : events) { 1942 switch (event) { 1943 case PROGRESS: 1944 numConsecutiveErrors = 0; 1945 break; 1946 case FAIL_CONNECTION_REFUSED: 1947 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, 1948 mockSleepProvider, numConsecutiveErrors); 1949 break; 1950 case FAIL_LINK_KILLED: 1951 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, 1952 mockSleepProvider, numConsecutiveErrors); 1953 break; 1954 } 1955 } 1956 } 1957 1958 /** 1959 * Test dEQP runner recovery state machine. 1960 */ testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)1961 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) 1962 throws Exception { 1963 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1964 IMocksControl orderedControl = EasyMock.createStrictControl(); 1965 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1966 DeqpTestRunner.ISleepProvider mockSleepProvider = 1967 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1968 1969 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); 1970 1971 orderedControl.replay(); 1972 1973 recovery.setDevice(mockDevice); 1974 recovery.setSleepProvider(mockSleepProvider); 1975 try { 1976 runRecoveryWithPattern(recovery, pattern); 1977 if (!expectSuccess) { 1978 fail("Expected DeviceNotAvailableException"); 1979 } 1980 } catch (DeviceNotAvailableException ex) { 1981 if (expectSuccess) { 1982 fail("Did not expect DeviceNotAvailableException"); 1983 } 1984 } 1985 1986 orderedControl.verify(); 1987 } 1988 1989 // basic patterns 1990 testRecovery_NoEvents()1991 public void testRecovery_NoEvents() throws Exception { 1992 testRecoveryWithPattern(true); 1993 } 1994 testRecovery_AllOk()1995 public void testRecovery_AllOk() throws Exception { 1996 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); 1997 } 1998 1999 // conn fail patterns 2000 testRecovery_OneConnectionFailureBegin()2001 public void testRecovery_OneConnectionFailureBegin() throws Exception { 2002 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2003 RecoveryEvent.PROGRESS); 2004 } 2005 testRecovery_TwoConnectionFailuresBegin()2006 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 2007 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2008 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 2009 } 2010 testRecovery_ThreeConnectionFailuresBegin()2011 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 2012 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2013 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); 2014 } 2015 testRecovery_OneConnectionFailureMid()2016 public void testRecovery_OneConnectionFailureMid() throws Exception { 2017 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 2018 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 2019 } 2020 testRecovery_TwoConnectionFailuresMid()2021 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 2022 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 2023 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2024 RecoveryEvent.PROGRESS); 2025 } 2026 testRecovery_ThreeConnectionFailuresMid()2027 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 2028 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 2029 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2030 RecoveryEvent.FAIL_CONNECTION_REFUSED); 2031 } 2032 2033 // link fail patterns 2034 testRecovery_OneLinkFailureBegin()2035 public void testRecovery_OneLinkFailureBegin() throws Exception { 2036 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 2037 RecoveryEvent.PROGRESS); 2038 } 2039 testRecovery_TwoLinkFailuresBegin()2040 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 2041 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 2042 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 2043 } 2044 testRecovery_ThreeLinkFailuresBegin()2045 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 2046 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 2047 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 2048 RecoveryEvent.PROGRESS); 2049 } 2050 testRecovery_FourLinkFailuresBegin()2051 public void testRecovery_FourLinkFailuresBegin() throws Exception { 2052 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 2053 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 2054 RecoveryEvent.FAIL_LINK_KILLED); 2055 } 2056 testRecovery_OneLinkFailureMid()2057 public void testRecovery_OneLinkFailureMid() throws Exception { 2058 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 2059 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 2060 } 2061 testRecovery_TwoLinkFailuresMid()2062 public void testRecovery_TwoLinkFailuresMid() throws Exception { 2063 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 2064 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 2065 RecoveryEvent.PROGRESS); 2066 } 2067 testRecovery_ThreeLinkFailuresMid()2068 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 2069 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 2070 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 2071 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 2072 } 2073 testRecovery_FourLinkFailuresMid()2074 public void testRecovery_FourLinkFailuresMid() throws Exception { 2075 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2076 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 2077 RecoveryEvent.FAIL_LINK_KILLED); 2078 } 2079 2080 // mixed patterns 2081 testRecovery_MixedFailuresProgressBetween()2082 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 2083 testRecoveryWithPattern(true, 2084 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2085 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2086 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2087 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 2088 RecoveryEvent.PROGRESS); 2089 } 2090 testRecovery_MixedFailuresNoProgressBetween()2091 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 2092 testRecoveryWithPattern(true, 2093 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 2094 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, 2095 RecoveryEvent.PROGRESS); 2096 } 2097 2098 /** 2099 * Test recovery if process cannot be killed 2100 */ testRecovery_unkillableProcess()2101 public void testRecovery_unkillableProcess () throws Exception { 2102 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 2103 IMocksControl orderedControl = EasyMock.createStrictControl(); 2104 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 2105 DeqpTestRunner.ISleepProvider mockSleepProvider = 2106 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 2107 2108 // recovery attempts to kill the process after a timeout 2109 mockSleepProvider.sleep(EasyMock.gt(0)); 2110 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 2111 andReturn("root 1234 com.drawelement.deqp").once(); 2112 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 2113 andReturn("").once(); 2114 2115 // Recovery checks if kill failed 2116 mockSleepProvider.sleep(EasyMock.gt(0)); 2117 EasyMock.expectLastCall().once(); 2118 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 2119 andReturn("root 1234 com.drawelement.deqp").once(); 2120 2121 // Recovery resets the connection 2122 mockDevice.recoverDevice(); 2123 EasyMock.expectLastCall().once(); 2124 2125 // and attempts to kill the process again 2126 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 2127 andReturn("root 1234 com.drawelement.deqp").once(); 2128 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 2129 andReturn("").once(); 2130 2131 // Recovery checks if kill failed 2132 mockSleepProvider.sleep(EasyMock.gt(0)); 2133 EasyMock.expectLastCall().once(); 2134 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 2135 andReturn("root 1234 com.drawelement.deqp").once(); 2136 2137 // recovery reboots the device 2138 mockDevice.reboot(); 2139 EasyMock.expectLastCall().once(); 2140 2141 orderedControl.replay(); 2142 recovery.setDevice(mockDevice); 2143 recovery.setSleepProvider(mockSleepProvider); 2144 recovery.recoverComLinkKilled(); 2145 orderedControl.verify(); 2146 } 2147 2148 /** 2149 * Test external interruption before batch run. 2150 */ testInterrupt_killBeforeBatch()2151 public void testInterrupt_killBeforeBatch() throws Exception { 2152 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test"); 2153 2154 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 2155 tests.add(testId); 2156 2157 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 2158 instance.put(testId, DEFAULT_INSTANCE_ARGS); 2159 2160 ITestInvocationListener mockListener 2161 = EasyMock.createStrictMock(ITestInvocationListener.class); 2162 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2163 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2164 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 2165 2166 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 2167 deqpTest.setAbi(UnitTests.ABI); 2168 deqpTest.setDevice(mockDevice); 2169 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 2170 deqpTest.setRunUtil(mockRunUtil); 2171 2172 int version = 3 << 16; 2173 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2174 .andReturn(Integer.toString(version)).atLeastOnce(); 2175 2176 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 2177 andReturn("").once(); 2178 2179 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 2180 EasyMock.eq(true), 2181 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 2182 .once(); 2183 2184 expectRenderConfigQuery(mockDevice, 2185 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 2186 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 2187 + "--deqp-gl-minor-version=0"); 2188 2189 mockRunUtil.sleep(0); 2190 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 2191 2192 mockListener.testRunStarted(ID, 1); 2193 EasyMock.expectLastCall().once(); 2194 2195 EasyMock.replay(mockDevice, mockIDevice); 2196 EasyMock.replay(mockListener); 2197 EasyMock.replay(mockRunUtil); 2198 try { 2199 deqpTest.run(mockListener); 2200 fail("expected RunInterruptedException"); 2201 } catch (RunInterruptedException ex) { 2202 // expected 2203 } 2204 EasyMock.verify(mockRunUtil); 2205 EasyMock.verify(mockListener); 2206 EasyMock.verify(mockDevice, mockIDevice); 2207 } 2208 2209 /** 2210 * Test external interruption in testFailed(). 2211 */ testInterrupt_killReportTestFailed()2212 public void testInterrupt_killReportTestFailed() throws Exception { 2213 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test"); 2214 final String testPath = "dEQP-GLES3.interrupt.test"; 2215 final String testTrie = "{dEQP-GLES3{interrupt{test}}}"; 2216 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 2217 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2218 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 2219 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2220 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 2221 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2222 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 2223 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2224 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 2225 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 2226 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 2227 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2228 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 2229 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2230 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 2231 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 2232 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2233 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 2234 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 2235 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 2236 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2237 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 2238 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2239 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 2240 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 2241 + "INSTRUMENTATION_CODE: 0\r\n"; 2242 2243 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); 2244 tests.add(testId); 2245 2246 Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); 2247 instance.put(testId, DEFAULT_INSTANCE_ARGS); 2248 2249 ITestInvocationListener mockListener 2250 = EasyMock.createStrictMock(ITestInvocationListener.class); 2251 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2252 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2253 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 2254 2255 DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); 2256 deqpTest.setAbi(UnitTests.ABI); 2257 deqpTest.setDevice(mockDevice); 2258 deqpTest.setBuildHelper(new StubCtsBuildHelper()); 2259 deqpTest.setRunUtil(mockRunUtil); 2260 2261 int version = 3 << 16; 2262 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2263 .andReturn(Integer.toString(version)).atLeastOnce(); 2264 2265 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). 2266 andReturn("").once(); 2267 2268 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), 2269 EasyMock.eq(true), 2270 EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) 2271 .once(); 2272 2273 expectRenderConfigQuery(mockDevice, 2274 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 2275 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 2276 + "--deqp-gl-minor-version=0"); 2277 2278 mockRunUtil.sleep(0); 2279 EasyMock.expectLastCall().once(); 2280 2281 String commandLine = String.format( 2282 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2283 + "--deqp-screen-rotation=unspecified " 2284 + "--deqp-surface-type=window " 2285 + "--deqp-log-images=disable " 2286 + "--deqp-watchdog=enable", 2287 CASE_LIST_FILE_NAME); 2288 2289 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 2290 output); 2291 2292 mockListener.testRunStarted(ID, 1); 2293 EasyMock.expectLastCall().once(); 2294 2295 mockListener.testStarted(EasyMock.eq(testId)); 2296 EasyMock.expectLastCall().once(); 2297 2298 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull()); 2299 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 2300 2301 EasyMock.replay(mockDevice, mockIDevice); 2302 EasyMock.replay(mockListener); 2303 EasyMock.replay(mockRunUtil); 2304 try { 2305 deqpTest.run(mockListener); 2306 fail("expected RunInterruptedException"); 2307 } catch (RunInterruptedException ex) { 2308 // expected 2309 } 2310 EasyMock.verify(mockRunUtil); 2311 EasyMock.verify(mockListener); 2312 EasyMock.verify(mockDevice, mockIDevice); 2313 } 2314 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, final String cmd, final String output)2315 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2316 final String testTrie, final String cmd, final String output) throws Exception { 2317 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) 2318 .andReturn("").once(); 2319 2320 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) 2321 .andReturn("").once(); 2322 2323 EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)) 2324 .andReturn(true).once(); 2325 2326 String command = String.format( 2327 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" " 2328 + "-e deqpLogData \"%s\" %s", 2329 AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, cmd, false, 2330 INSTRUMENTATION_NAME); 2331 2332 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 2333 mockIDevice.executeShellCommand(EasyMock.eq(command), 2334 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 2335 EasyMock.isA(TimeUnit.class)); 2336 2337 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 2338 @Override 2339 public Object answer() { 2340 IShellOutputReceiver receiver 2341 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 2342 2343 receiver.addOutput(output.getBytes(), 0, output.length()); 2344 receiver.flush(); 2345 2346 return null; 2347 } 2348 }); 2349 } 2350 } 2351