1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.util; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertTrue; 21 import static org.junit.Assert.fail; 22 23 import com.android.tradefed.invoker.IInvocationContext; 24 import com.android.tradefed.invoker.InvocationContext; 25 import com.android.tradefed.log.LogUtil.CLog; 26 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 27 import com.android.tradefed.result.ILogSaverListener; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.result.LogDataType; 30 import com.android.tradefed.result.LogFile; 31 import com.android.tradefed.result.TestDescription; 32 33 import org.easymock.Capture; 34 import org.easymock.EasyMock; 35 import org.junit.Test; 36 import org.junit.runner.RunWith; 37 import org.junit.runners.JUnit4; 38 39 import java.io.BufferedReader; 40 import java.io.File; 41 import java.io.IOException; 42 import java.io.InputStream; 43 import java.io.InputStreamReader; 44 import java.io.PrintWriter; 45 import java.net.Socket; 46 import java.util.HashMap; 47 import java.util.Vector; 48 49 /** Unit Tests for {@link SubprocessTestResultsParser} */ 50 @RunWith(JUnit4.class) 51 public class SubprocessTestResultsParserTest { 52 53 private static final String TEST_TYPE_DIR = "testdata"; 54 private static final String SUBPROC_OUTPUT_FILE_1 = "subprocess1.txt"; 55 private static final String SUBPROC_OUTPUT_FILE_2 = "subprocess2.txt"; 56 57 /** 58 * Helper to read a file from the res/testdata directory and return its contents as a String[] 59 * 60 * @param filename the name of the file (without the extension) in the res/testdata directory 61 * @return a String[] of the 62 */ readInFile(String filename)63 private String[] readInFile(String filename) { 64 Vector<String> fileContents = new Vector<String>(); 65 try { 66 InputStream gtestResultStream1 = getClass().getResourceAsStream(File.separator + 67 TEST_TYPE_DIR + File.separator + filename); 68 BufferedReader reader = new BufferedReader(new InputStreamReader(gtestResultStream1)); 69 String line = null; 70 while ((line = reader.readLine()) != null) { 71 fileContents.add(line); 72 } 73 } 74 catch (NullPointerException e) { 75 CLog.e("Gest output file does not exist: " + filename); 76 } 77 catch (IOException e) { 78 CLog.e("Unable to read contents of gtest output file: " + filename); 79 } 80 return fileContents.toArray(new String[fileContents.size()]); 81 } 82 83 /** Tests the parser for cases of test failed, ignored, assumption failure */ 84 @Test testParse_randomEvents()85 public void testParse_randomEvents() throws Exception { 86 String[] contents = readInFile(SUBPROC_OUTPUT_FILE_1); 87 ITestInvocationListener mockRunListener = 88 EasyMock.createMock(ITestInvocationListener.class); 89 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 90 mockRunListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 91 EasyMock.expectLastCall().times(4); 92 mockRunListener.testEnded( 93 (TestDescription) EasyMock.anyObject(), 94 EasyMock.anyLong(), 95 (HashMap<String, Metric>) EasyMock.anyObject()); 96 EasyMock.expectLastCall().times(4); 97 mockRunListener.testRunEnded( 98 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 99 EasyMock.expectLastCall().times(1); 100 mockRunListener.testIgnored((TestDescription) EasyMock.anyObject()); 101 EasyMock.expectLastCall(); 102 mockRunListener.testFailed( 103 (TestDescription) EasyMock.anyObject(), (String) EasyMock.anyObject()); 104 EasyMock.expectLastCall(); 105 mockRunListener.testAssumptionFailure( 106 (TestDescription) EasyMock.anyObject(), (String) EasyMock.anyObject()); 107 EasyMock.expectLastCall(); 108 EasyMock.replay(mockRunListener); 109 File tmp = FileUtil.createTempFile("sub", "unit"); 110 SubprocessTestResultsParser resultParser = null; 111 try { 112 resultParser = 113 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 114 resultParser.processNewLines(contents); 115 EasyMock.verify(mockRunListener); 116 } finally { 117 StreamUtil.close(resultParser); 118 FileUtil.deleteFile(tmp); 119 } 120 } 121 122 /** Tests the parser for cases of test starting without closing. */ 123 @Test testParse_invalidEventOrder()124 public void testParse_invalidEventOrder() throws Exception { 125 String[] contents = readInFile(SUBPROC_OUTPUT_FILE_2); 126 ITestInvocationListener mockRunListener = 127 EasyMock.createMock(ITestInvocationListener.class); 128 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 129 mockRunListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 130 EasyMock.expectLastCall().times(4); 131 mockRunListener.testEnded( 132 (TestDescription) EasyMock.anyObject(), 133 EasyMock.anyLong(), 134 (HashMap<String, Metric>) EasyMock.anyObject()); 135 EasyMock.expectLastCall().times(3); 136 mockRunListener.testRunFailed((String)EasyMock.anyObject()); 137 EasyMock.expectLastCall().times(1); 138 mockRunListener.testRunEnded( 139 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 140 EasyMock.expectLastCall().times(1); 141 mockRunListener.testIgnored((TestDescription) EasyMock.anyObject()); 142 EasyMock.expectLastCall(); 143 mockRunListener.testAssumptionFailure( 144 (TestDescription) EasyMock.anyObject(), (String) EasyMock.anyObject()); 145 EasyMock.expectLastCall(); 146 EasyMock.replay(mockRunListener); 147 File tmp = FileUtil.createTempFile("sub", "unit"); 148 SubprocessTestResultsParser resultParser = null; 149 try { 150 resultParser = 151 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 152 resultParser.processNewLines(contents); 153 EasyMock.verify(mockRunListener); 154 } finally { 155 StreamUtil.close(resultParser); 156 FileUtil.deleteFile(tmp); 157 } 158 } 159 160 /** Tests the parser for cases of test starting without closing. */ 161 @Test testParse_testNotStarted()162 public void testParse_testNotStarted() throws Exception { 163 ITestInvocationListener mockRunListener = 164 EasyMock.createMock(ITestInvocationListener.class); 165 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 166 mockRunListener.testEnded( 167 (TestDescription) EasyMock.anyObject(), 168 EasyMock.anyLong(), 169 (HashMap<String, Metric>) EasyMock.anyObject()); 170 EasyMock.expectLastCall().times(1); 171 EasyMock.replay(mockRunListener); 172 File tmp = FileUtil.createTempFile("sub", "unit"); 173 SubprocessTestResultsParser resultParser = null; 174 try { 175 resultParser = 176 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 177 String startRun = 178 "TEST_RUN_STARTED {\"testCount\":4,\"runName\":\"arm64-v8a " 179 + "CtsGestureTestCases\"}\n"; 180 FileUtil.writeToFile(startRun, tmp, true); 181 String testEnded = 182 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_ENDED " 183 + "{\"end_time\":1489160958359,\"className\":\"android.gesture.cts." 184 + "GestureLibraryTest\",\"testName\":\"testGetGestures\",\"extra\":\"" 185 + "data\"}\n"; 186 FileUtil.writeToFile(testEnded, tmp, true); 187 resultParser.parseFile(tmp); 188 EasyMock.verify(mockRunListener); 189 } finally { 190 StreamUtil.close(resultParser); 191 FileUtil.deleteFile(tmp); 192 } 193 } 194 195 /** Tests the parser for a cases when there is no start/end time stamp. */ 196 @Test testParse_noTimeStamp()197 public void testParse_noTimeStamp() throws Exception { 198 ITestInvocationListener mockRunListener = 199 EasyMock.createMock(ITestInvocationListener.class); 200 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 201 mockRunListener.testStarted(EasyMock.anyObject()); 202 mockRunListener.testEnded( 203 (TestDescription) EasyMock.anyObject(), 204 (HashMap<String, Metric>) EasyMock.anyObject()); 205 EasyMock.expectLastCall().times(1); 206 EasyMock.replay(mockRunListener); 207 File tmp = FileUtil.createTempFile("sub", "unit"); 208 SubprocessTestResultsParser resultParser = null; 209 try { 210 resultParser = 211 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 212 String startRun = "TEST_RUN_STARTED {\"testCount\":4,\"runName\":\"arm64-v8a " 213 + "CtsGestureTestCases\"}\n"; 214 FileUtil.writeToFile(startRun, tmp, true); 215 String testStarted = 216 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_STARTED " 217 + "{\"className\":\"android.gesture.cts." 218 + "GestureLibraryTest\",\"testName\":\"testGetGestures\"}\n"; 219 FileUtil.writeToFile(testStarted, tmp, true); 220 String testEnded = 221 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_ENDED " 222 + "{\"className\":\"android.gesture.cts." 223 + "GestureLibraryTest\",\"testName\":\"testGetGestures\",\"extra\":\"" 224 + "data\"}\n"; 225 FileUtil.writeToFile(testEnded, tmp, true); 226 resultParser.parseFile(tmp); 227 EasyMock.verify(mockRunListener); 228 } finally { 229 StreamUtil.close(resultParser); 230 FileUtil.deleteFile(tmp); 231 } 232 } 233 234 /** Test injecting an invocation failure and verify the callback is called. */ 235 @Test testParse_invocationFailed()236 public void testParse_invocationFailed() throws Exception { 237 ITestInvocationListener mockRunListener = 238 EasyMock.createMock(ITestInvocationListener.class); 239 Capture<Throwable> cap = new Capture<Throwable>(); 240 mockRunListener.invocationFailed((EasyMock.capture(cap))); 241 EasyMock.replay(mockRunListener); 242 File tmp = FileUtil.createTempFile("sub", "unit"); 243 SubprocessTestResultsParser resultParser = null; 244 try { 245 resultParser = 246 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 247 String cause = "com.android.tradefed.targetprep." 248 + "TargetSetupError: Not all target preparation steps completed\n\tat " 249 + "com.android.compatibility.common.tradefed.targetprep." 250 + "ApkInstrumentationPreparer.run(ApkInstrumentationPreparer.java:88)\n"; 251 String startRun = "03-23 11:50:12 E/SubprocessResultsReporter: " 252 + "INVOCATION_FAILED {\"cause\":\"com.android.tradefed.targetprep." 253 + "TargetSetupError: Not all target preparation steps completed\\n\\tat " 254 + "com.android.compatibility.common.tradefed.targetprep." 255 + "ApkInstrumentationPreparer.run(ApkInstrumentationPreparer.java:88)\\n\"}\n"; 256 FileUtil.writeToFile(startRun, tmp, true); 257 resultParser.parseFile(tmp); 258 EasyMock.verify(mockRunListener); 259 String expected = cap.getValue().getMessage(); 260 assertEquals(cause, expected); 261 } finally { 262 StreamUtil.close(resultParser); 263 FileUtil.deleteFile(tmp); 264 } 265 } 266 267 /** Report results when received from socket. */ 268 @Test testParser_receiveFromSocket()269 public void testParser_receiveFromSocket() throws Exception { 270 ITestInvocationListener mockRunListener = 271 EasyMock.createMock(ITestInvocationListener.class); 272 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 273 mockRunListener.testEnded( 274 (TestDescription) EasyMock.anyObject(), 275 EasyMock.anyLong(), 276 (HashMap<String, Metric>) EasyMock.anyObject()); 277 EasyMock.expectLastCall().times(1); 278 EasyMock.replay(mockRunListener); 279 SubprocessTestResultsParser resultParser = null; 280 Socket socket = null; 281 try { 282 resultParser = 283 new SubprocessTestResultsParser(mockRunListener, true, new InvocationContext()); 284 socket = new Socket("localhost", resultParser.getSocketServerPort()); 285 if (!socket.isConnected()) { 286 fail("socket did not connect"); 287 } 288 PrintWriter out = new PrintWriter(socket.getOutputStream(), true); 289 String startRun = "TEST_RUN_STARTED {\"testCount\":4,\"runName\":\"arm64-v8a " 290 + "CtsGestureTestCases\"}\n"; 291 out.print(startRun); 292 out.flush(); 293 String testEnded = 294 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_ENDED " 295 + "{\"end_time\":1489160958359,\"className\":\"android.gesture.cts." 296 + "GestureLibraryTest\",\"testName\":\"testGetGestures\",\"extra\":\"" 297 + "data\"}\n"; 298 out.print(testEnded); 299 out.flush(); 300 StreamUtil.close(socket); 301 assertTrue(resultParser.joinReceiver(500)); 302 EasyMock.verify(mockRunListener); 303 } finally { 304 StreamUtil.close(resultParser); 305 StreamUtil.close(socket); 306 } 307 } 308 309 /** When the receiver thread fails to join then an exception is thrown. */ 310 @Test testParser_failToJoin()311 public void testParser_failToJoin() throws Exception { 312 ITestInvocationListener mockRunListener = 313 EasyMock.createMock(ITestInvocationListener.class); 314 EasyMock.replay(mockRunListener); 315 SubprocessTestResultsParser resultParser = null; 316 try { 317 resultParser = 318 new SubprocessTestResultsParser(mockRunListener, true, new InvocationContext()); 319 assertFalse(resultParser.joinReceiver(50)); 320 EasyMock.verify(mockRunListener); 321 } finally { 322 StreamUtil.close(resultParser); 323 } 324 } 325 326 /** Tests the parser receiving event on updating test tag. */ 327 @Test testParse_testTag()328 public void testParse_testTag() throws Exception { 329 final String subTestTag = "test_tag_in_subprocess"; 330 InvocationContext context = new InvocationContext(); 331 context.setTestTag("stub"); 332 333 ITestInvocationListener mockRunListener = 334 EasyMock.createMock(ITestInvocationListener.class); 335 EasyMock.replay(mockRunListener); 336 File tmp = FileUtil.createTempFile("sub", "unit"); 337 SubprocessTestResultsParser resultParser = null; 338 try { 339 resultParser = new SubprocessTestResultsParser(mockRunListener, false, context); 340 String testTagEvent = 341 String.format( 342 "INVOCATION_STARTED {\"testTag\": \"%s\",\"start_time\":250}", 343 subTestTag); 344 FileUtil.writeToFile(testTagEvent, tmp, true); 345 resultParser.parseFile(tmp); 346 EasyMock.verify(mockRunListener); 347 assertEquals(subTestTag, context.getTestTag()); 348 assertEquals(250l, resultParser.getStartTime().longValue()); 349 } finally { 350 StreamUtil.close(resultParser); 351 FileUtil.deleteFile(tmp); 352 } 353 } 354 355 /** Tests the parser should not overwrite the test tag in parent process if it's already set. */ 356 @Test testParse_testTagNotOverwrite()357 public void testParse_testTagNotOverwrite() throws Exception { 358 final String subTestTag = "test_tag_in_subprocess"; 359 final String parentTestTag = "test_tag_in_parent_process"; 360 InvocationContext context = new InvocationContext(); 361 context.setTestTag(parentTestTag); 362 363 ITestInvocationListener mockRunListener = 364 EasyMock.createMock(ITestInvocationListener.class); 365 EasyMock.replay(mockRunListener); 366 File tmp = FileUtil.createTempFile("sub", "unit"); 367 SubprocessTestResultsParser resultParser = null; 368 try { 369 resultParser = new SubprocessTestResultsParser(mockRunListener, false, context); 370 String testTagEvent = String.format("TEST_TAG %s", subTestTag); 371 FileUtil.writeToFile(testTagEvent, tmp, true); 372 resultParser.parseFile(tmp); 373 EasyMock.verify(mockRunListener); 374 assertEquals(parentTestTag, context.getTestTag()); 375 } finally { 376 StreamUtil.close(resultParser); 377 FileUtil.deleteFile(tmp); 378 } 379 } 380 381 /** Test that module start and end is properly parsed when reported. */ 382 @Test testParse_moduleStarted_end()383 public void testParse_moduleStarted_end() throws Exception { 384 ITestInvocationListener mockRunListener = 385 EasyMock.createMock(ITestInvocationListener.class); 386 mockRunListener.testModuleStarted(EasyMock.anyObject()); 387 mockRunListener.testModuleEnded(); 388 EasyMock.replay(mockRunListener); 389 IInvocationContext fakeModuleContext = new InvocationContext(); 390 File tmp = FileUtil.createTempFile("sub", "unit"); 391 SubprocessTestResultsParser resultParser = null; 392 File serializedModule = null; 393 try { 394 serializedModule = SerializationUtil.serialize(fakeModuleContext); 395 resultParser = 396 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 397 String moduleStart = 398 String.format( 399 "TEST_MODULE_STARTED {\"moduleContextFileName\":\"%s\"}\n", 400 serializedModule.getAbsolutePath()); 401 FileUtil.writeToFile(moduleStart, tmp, true); 402 String moduleEnd = "TEST_MODULE_ENDED {}\n"; 403 FileUtil.writeToFile(moduleEnd, tmp, true); 404 405 resultParser.parseFile(tmp); 406 EasyMock.verify(mockRunListener); 407 } finally { 408 StreamUtil.close(resultParser); 409 FileUtil.deleteFile(tmp); 410 FileUtil.deleteFile(serializedModule); 411 } 412 } 413 414 /** Test that logAssociation event is properly passed and parsed. */ 415 @Test testParse_logAssociation()416 public void testParse_logAssociation() throws Exception { 417 ILogSaverListener mockRunListener = EasyMock.createMock(ILogSaverListener.class); 418 Capture<LogFile> capture = new Capture<>(); 419 mockRunListener.logAssociation(EasyMock.eq("dataname"), EasyMock.capture(capture)); 420 EasyMock.replay(mockRunListener); 421 LogFile logFile = new LogFile("path", "url", LogDataType.TEXT); 422 File serializedLogFile = null; 423 File tmp = FileUtil.createTempFile("sub", "unit"); 424 SubprocessTestResultsParser resultParser = null; 425 try { 426 serializedLogFile = SerializationUtil.serialize(logFile); 427 resultParser = 428 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 429 String logAssocation = 430 String.format( 431 "LOG_ASSOCIATION {\"loggedFile\":\"%s\",\"dataName\":\"dataname\"}\n", 432 serializedLogFile.getAbsolutePath()); 433 FileUtil.writeToFile(logAssocation, tmp, true); 434 resultParser.parseFile(tmp); 435 EasyMock.verify(mockRunListener); 436 } finally { 437 StreamUtil.close(resultParser); 438 FileUtil.deleteFile(serializedLogFile); 439 FileUtil.deleteFile(tmp); 440 } 441 LogFile received = capture.getValue(); 442 assertEquals(logFile.getPath(), received.getPath()); 443 assertEquals(logFile.getUrl(), received.getUrl()); 444 assertEquals(logFile.getType(), received.getType()); 445 } 446 } 447