1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.device.collectors; 17 18 import android.app.Instrumentation; 19 import android.device.collectors.annotations.MetricOption; 20 import android.device.collectors.annotations.OptionClass; 21 import android.device.collectors.util.SendToInstrumentation; 22 import android.os.Bundle; 23 24 import androidx.test.runner.AndroidJUnit4; 25 26 import org.junit.Before; 27 import org.junit.Test; 28 import org.junit.runner.Description; 29 import org.junit.runner.Result; 30 import org.junit.runner.RunWith; 31 import org.mockito.ArgumentCaptor; 32 import org.mockito.Mockito; 33 34 import java.lang.annotation.Annotation; 35 import java.util.List; 36 import java.util.Objects; 37 38 import static org.junit.Assert.assertEquals; 39 import static org.junit.Assert.assertFalse; 40 import static org.junit.Assert.assertTrue; 41 42 /** 43 * Android Unit Tests for {@link BaseMetricListener}. 44 */ 45 @RunWith(AndroidJUnit4.class) 46 public class BaseMetricListenerInstrumentedTest { 47 48 private static final String RUN_START_KEY = "run_start_key"; 49 private static final String RUN_END_KEY = "run_end_key"; 50 private static final String RUN_START_VALUE = "run_start_value"; 51 private static final String RUN_END_VALUE = "run_end_value"; 52 private static final String TEST_START_KEY = "test_start_key"; 53 private static final String TEST_END_KEY = "test_end_key"; 54 private static final String TEST_START_VALUE = "test_start_value"; 55 private static final String TEST_END_VALUE = "test_end_value"; 56 private static final String SKIP_METRICS_OPTION = "skip-metrics"; 57 private BaseMetricListener mListener; 58 private Instrumentation mMockInstrumentation; 59 60 @Before setUp()61 public void setUp() { 62 mMockInstrumentation = Mockito.mock(Instrumentation.class); 63 mListener = createWithArgs(null); 64 mListener.setInstrumentation(mMockInstrumentation); 65 } 66 createWithArgs(Bundle args)67 private BaseMetricListener createWithArgs(Bundle args) { 68 if (args == null) { 69 args = new Bundle(); 70 } 71 return new BaseMetricListener(args) { 72 private boolean mSuppressMetrics = false; 73 74 @Override 75 public void onTestStart(DataRecord testData, Description description) { 76 // In this test check that a new DataRecord is passed to testStart each time. 77 assertFalse(testData.hasMetrics()); 78 if (!mSuppressMetrics) { 79 testData.addStringMetric( 80 TEST_START_KEY, TEST_START_VALUE + description.getMethodName()); 81 } 82 } 83 84 @Override 85 public void onTestEnd(DataRecord testData, Description description) { 86 if (!mSuppressMetrics) { 87 testData.addStringMetric( 88 TEST_END_KEY, TEST_END_VALUE + description.getMethodName()); 89 } 90 } 91 92 @Override 93 public void onTestRunStart(DataRecord runData, Description description) { 94 assertFalse(runData.hasMetrics()); 95 if (!mSuppressMetrics) { 96 runData.addStringMetric(RUN_START_KEY, RUN_START_VALUE); 97 } 98 } 99 100 @Override 101 public void onTestRunEnd(DataRecord runData, Result result) { 102 if (!mSuppressMetrics) { 103 runData.addStringMetric(RUN_END_KEY, RUN_END_VALUE); 104 } 105 } 106 107 @Override 108 public void setupAdditionalArgs() { 109 // We test this method by checking if metrics are suppressed when the skip metrics 110 // option is set. 111 mSuppressMetrics = 112 Boolean.parseBoolean(getArgsBundle().getString(SKIP_METRICS_OPTION)); 113 } 114 }; 115 } 116 117 /** 118 * When metrics are logged during a test, expect them to be added to the bundle. 119 */ 120 @MetricOption(group = "testGroup,testGroup1") 121 @Test testReportMetrics()122 public void testReportMetrics() throws Exception { 123 Description runDescription = Description.createSuiteDescription("run"); 124 mListener.testRunStarted(runDescription); 125 Description testDescription = Description.createTestDescription("class", "method"); 126 mListener.testStarted(testDescription); 127 mListener.testFinished(testDescription); 128 mListener.testRunFinished(new Result()); 129 // AJUR runner is then gonna call instrumentationRunFinished 130 Bundle resultBundle = new Bundle(); 131 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 132 133 // Check that the in progress status contains the metrics. 134 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 135 Mockito.verify(mMockInstrumentation) 136 .sendStatus(Mockito.eq( 137 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 138 List<Bundle> capturedBundle = capture.getAllValues(); 139 assertEquals(1, capturedBundle.size()); 140 Bundle check = capturedBundle.get(0); 141 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 142 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 143 assertEquals(2, check.size()); 144 145 // Check that final bundle contains run results 146 assertEquals(RUN_START_VALUE, resultBundle.getString(RUN_START_KEY)); 147 assertEquals(RUN_END_VALUE, resultBundle.getString(RUN_END_KEY)); 148 assertEquals(2, resultBundle.size()); 149 } 150 151 /** 152 * Test that only included group are running collection. 153 */ 154 @MetricOption(group = "testGroup") 155 @Test testReportMetrics_withIncludeFilters()156 public void testReportMetrics_withIncludeFilters() throws Exception { 157 Bundle args = new Bundle(); 158 args.putString(BaseMetricListener.INCLUDE_FILTER_GROUP_KEY, "group1,group2"); 159 mListener = createWithArgs(args); 160 mListener.setInstrumentation(mMockInstrumentation); 161 162 Description runDescription = Description.createSuiteDescription("run"); 163 mListener.testRunStarted(runDescription); 164 Description testDescription = Description.createTestDescription("class", "method", 165 new TestAnnotation("group1")); 166 mListener.testStarted(testDescription); 167 mListener.testFinished(testDescription); 168 // A second test case that will not be included 169 Description testDescription2 = Description.createTestDescription("class", "method2", 170 new TestAnnotation("group3")); 171 mListener.testStarted(testDescription2); 172 mListener.testFinished(testDescription2); 173 // A third test case that will be included 174 Description testDescription3 = Description.createTestDescription("class", "method3", 175 new TestAnnotation("group2")); 176 mListener.testStarted(testDescription3); 177 mListener.testFinished(testDescription3); 178 179 // Check that the in progress status contains the metrics. 180 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 181 Mockito.verify(mMockInstrumentation, Mockito.times(2)) 182 .sendStatus(Mockito.eq( 183 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 184 // Check that only method2 did not generate any metrics since it was filtered. 185 List<Bundle> capturedBundle = capture.getAllValues(); 186 assertEquals(2, capturedBundle.size()); 187 Bundle check = capturedBundle.get(0); 188 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 189 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 190 assertEquals(2, check.size()); 191 // Got a call from the method3 192 Bundle check2 = capturedBundle.get(1); 193 assertEquals(TEST_START_VALUE + "method3", check2.getString(TEST_START_KEY)); 194 assertEquals(TEST_END_VALUE + "method3", check2.getString(TEST_END_KEY)); 195 assertEquals(2, check2.size()); 196 } 197 198 /** 199 * Test that only included group are running collection, even if some method have multiple 200 * groups, if any of its group is included, the method is included. 201 */ 202 @MetricOption(group = "testGroup") 203 @Test testReportMetrics_withIncludeFilters_multiGroup()204 public void testReportMetrics_withIncludeFilters_multiGroup() throws Exception { 205 Bundle args = new Bundle(); 206 args.putString(BaseMetricListener.INCLUDE_FILTER_GROUP_KEY, "group4"); 207 mListener = createWithArgs(args); 208 mListener.setInstrumentation(mMockInstrumentation); 209 210 Description runDescription = Description.createSuiteDescription("run"); 211 mListener.testRunStarted(runDescription); 212 Description testDescription = Description.createTestDescription("class", "method", 213 new TestAnnotation("group1")); 214 mListener.testStarted(testDescription); 215 mListener.testFinished(testDescription); 216 // A second test case that will not be included 217 Description testDescription2 = Description.createTestDescription("class", "method2", 218 new TestAnnotation("group3,group4")); 219 mListener.testStarted(testDescription2); 220 mListener.testFinished(testDescription2); 221 // A third test case that will be included 222 Description testDescription3 = Description.createTestDescription("class", "method3", 223 new TestAnnotation("group2")); 224 mListener.testStarted(testDescription3); 225 mListener.testFinished(testDescription3); 226 227 // Check that the in progress status contains the metrics. 228 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 229 Mockito.verify(mMockInstrumentation, Mockito.times(1)) 230 .sendStatus(Mockito.eq( 231 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 232 // Check that only method2 did generate metrics since it was included. 233 List<Bundle> capturedBundle = capture.getAllValues(); 234 assertEquals(1, capturedBundle.size()); 235 Bundle check = capturedBundle.get(0); 236 // Got call from method2 237 assertEquals(TEST_START_VALUE + "method2", check.getString(TEST_START_KEY)); 238 assertEquals(TEST_END_VALUE + "method2", check.getString(TEST_END_KEY)); 239 assertEquals(2, check.size()); 240 } 241 242 /** 243 * Test that only not excluded group are running collection. 244 */ 245 @MetricOption(group = "testGroup") 246 @Test testReportMetrics_withExcludeFilters()247 public void testReportMetrics_withExcludeFilters() throws Exception { 248 Bundle args = new Bundle(); 249 args.putString(BaseMetricListener.EXCLUDE_FILTER_GROUP_KEY, "group1,group2"); 250 mListener = createWithArgs(args); 251 mListener.setInstrumentation(mMockInstrumentation); 252 253 Description runDescription = Description.createSuiteDescription("run"); 254 mListener.testRunStarted(runDescription); 255 // A first test case that will not be included 256 Description testDescription = Description.createTestDescription("class", "method", 257 new TestAnnotation("group1")); 258 mListener.testStarted(testDescription); 259 mListener.testFinished(testDescription); 260 // A second test case that will run 261 Description testDescription2 = Description.createTestDescription("class", "method2", 262 new TestAnnotation("group3")); 263 mListener.testStarted(testDescription2); 264 mListener.testFinished(testDescription2); 265 // A third test case that will not be included 266 Description testDescription3 = Description.createTestDescription("class", "method3", 267 new TestAnnotation("group2")); 268 mListener.testStarted(testDescription3); 269 mListener.testFinished(testDescription3); 270 271 // Check that the in progress status contains the metrics. 272 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 273 Mockito.verify(mMockInstrumentation, Mockito.times(1)) 274 .sendStatus(Mockito.eq( 275 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 276 // Check that only method2 generates some metrics 277 List<Bundle> capturedBundle = capture.getAllValues(); 278 assertEquals(1, capturedBundle.size()); 279 Bundle check = capturedBundle.get(0); 280 assertEquals(TEST_START_VALUE + "method2", check.getString(TEST_START_KEY)); 281 assertEquals(TEST_END_VALUE + "method2", check.getString(TEST_END_KEY)); 282 assertEquals(2, check.size()); 283 } 284 285 /** 286 * Test that when both filters are present, exclude filters have priority. 287 */ 288 @MetricOption(group = "testGroup") 289 @Test testReportMetrics_withBothFilters()290 public void testReportMetrics_withBothFilters() throws Exception { 291 Bundle args = new Bundle(); 292 args.putString(BaseMetricListener.EXCLUDE_FILTER_GROUP_KEY, "group1,group2"); 293 args.putString(BaseMetricListener.INCLUDE_FILTER_GROUP_KEY, "group2,group3"); 294 mListener = createWithArgs(args); 295 mListener.setInstrumentation(mMockInstrumentation); 296 297 Description runDescription = Description.createSuiteDescription("run"); 298 mListener.testRunStarted(runDescription); 299 // A first test case that will not be included 300 Description testDescription = Description.createTestDescription("class", "method", 301 new TestAnnotation("group1")); 302 mListener.testStarted(testDescription); 303 mListener.testFinished(testDescription); 304 // A second test case that will run 305 Description testDescription2 = Description.createTestDescription("class", "method2", 306 new TestAnnotation("group3")); 307 mListener.testStarted(testDescription2); 308 mListener.testFinished(testDescription2); 309 // A third test case that will not be included 310 Description testDescription3 = Description.createTestDescription("class", "method3", 311 new TestAnnotation("group2")); 312 mListener.testStarted(testDescription3); 313 mListener.testFinished(testDescription3); 314 315 // Check that the in progress status contains the metrics. 316 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 317 Mockito.verify(mMockInstrumentation, Mockito.times(1)) 318 .sendStatus(Mockito.eq( 319 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 320 // Check that only method2 generates some metrics 321 List<Bundle> capturedBundle = capture.getAllValues(); 322 assertEquals(1, capturedBundle.size()); 323 Bundle check = capturedBundle.get(0); 324 assertEquals(TEST_START_VALUE + "method2", check.getString(TEST_START_KEY)); 325 assertEquals(TEST_END_VALUE + "method2", check.getString(TEST_END_KEY)); 326 assertEquals(2, check.size()); 327 } 328 329 /** Test that the parsing of the skip metrics option in setUpAdditionalArgs() is executed. */ 330 @MetricOption(group = "testGroup") 331 @Test testAdditionalArgs()332 public void testAdditionalArgs() throws Exception { 333 Bundle args = new Bundle(); 334 args.putString(SKIP_METRICS_OPTION, String.valueOf(true)); 335 mListener = createWithArgs(args); 336 mListener.setInstrumentation(mMockInstrumentation); 337 338 Description runDescription = Description.createSuiteDescription("run"); 339 mListener.testRunStarted(runDescription); 340 Description testDescription = Description.createTestDescription("class", "method"); 341 mListener.testStarted(testDescription); 342 mListener.testFinished(testDescription); 343 mListener.testRunFinished(new Result()); 344 // AJUR runner is then gonna call instrumentationRunFinished 345 Bundle resultBundle = new Bundle(); 346 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 347 348 Mockito.verify(mMockInstrumentation, Mockito.never()) 349 .sendStatus(Mockito.anyInt(), Mockito.any()); 350 351 // We have skipped reporting metrics, so no run metrics should be in the bundle either. 352 assertEquals(0, resultBundle.size()); 353 } 354 355 @MetricOption(group = "testGroup") 356 @Test testSetUpAndCleanUpWithTestCycle()357 public void testSetUpAndCleanUpWithTestCycle() throws Exception { 358 // We use this bundle to mimic device state. We modify it in setUp() and 359 // check that this is maintained throughout the test. We remove the 360 // modification during cleanUp() and check if the bundle has indeed 361 // become empty again. 362 Bundle data = new Bundle(); 363 final String arg = "arg"; 364 final String value = "value"; 365 366 BaseMetricListener listener = 367 new BaseMetricListener() { 368 public static final String SETUP_ARG = "arg"; 369 private static final String SETUP_VALUE = "value"; 370 371 @Override 372 protected void onSetUp() { 373 data.putString(arg, value); 374 } 375 376 @Override 377 protected void onCleanUp() { 378 data.remove(arg); 379 } 380 381 @Override 382 public void onTestRunStart(DataRecord runData, Description description) { 383 assertEquals(value, data.getString(arg)); 384 } 385 386 @Override 387 public void onTestRunEnd(DataRecord runData, Result result) { 388 assertEquals(value, data.getString(arg)); 389 } 390 391 @Override 392 public void onTestStart(DataRecord testData, Description description) { 393 assertEquals(value, data.getString(arg)); 394 } 395 396 @Override 397 public void onTestEnd(DataRecord testData, Description description) { 398 assertEquals(value, data.getString(arg)); 399 } 400 }; 401 402 Description runDescription = Description.createSuiteDescription("run"); 403 Description test1Description = Description.createTestDescription("class", "method1"); 404 Description test2Description = Description.createTestDescription("class", "method2"); 405 // Simulate a test cycle. 406 listener.testRunStarted(runDescription); 407 listener.testStarted(test1Description); 408 listener.testFinished(test1Description); 409 listener.testStarted(test2Description); 410 listener.testFinished(test2Description); 411 listener.testRunFinished(new Result()); 412 listener.instrumentationRunFinished(System.out, new Bundle(), new Result()); 413 414 assertFalse(data.containsKey(arg)); 415 } 416 417 @MetricOption(group = "testGroup") 418 @Test testSetUpAndCleanUpWithCallbacks()419 public void testSetUpAndCleanUpWithCallbacks() throws Exception { 420 // We use this bundle to mimic device state. We modify it in setUp() and 421 // check that this is maintained throughout the test. We remove the 422 // modification during cleanUp() and check if the bundle has indeed 423 // become empty again. 424 Bundle data = new Bundle(); 425 final String arg = "arg"; 426 final String value = "value"; 427 428 BaseMetricListener listener = 429 new BaseMetricListener() { 430 public static final String SETUP_ARG = "arg"; 431 private static final String SETUP_VALUE = "value"; 432 433 @Override 434 protected void onSetUp() { 435 data.putString(arg, value); 436 } 437 438 @Override 439 protected void onCleanUp() { 440 data.remove(arg); 441 } 442 }; 443 444 listener.setUp(); 445 assertEquals(value, data.getString(arg)); 446 listener.cleanUp(); 447 assertFalse(data.containsKey(arg)); 448 } 449 450 /** 451 * Test annotation that allows to instantiate {@link MetricOption} for testing purpose. 452 */ 453 public static class TestAnnotation implements MetricOption { 454 private String mGroup; 455 TestAnnotation(String group)456 public TestAnnotation(String group) { 457 mGroup = group; 458 } 459 460 @Override group()461 public String group() { 462 return mGroup; 463 } 464 465 @Override annotationType()466 public Class<? extends Annotation> annotationType() { 467 return MetricOption.class; 468 } 469 470 @Override equals(Object other)471 public boolean equals(Object other) { 472 if (this == other) { 473 return true; 474 } 475 if (!(other instanceof TestAnnotation)) { 476 return false; 477 } 478 TestAnnotation o = (TestAnnotation) other; 479 return Objects.equals(mGroup, o.mGroup); 480 } 481 482 @Override hashCode()483 public int hashCode() { 484 return Objects.hashCode(mGroup); 485 } 486 } 487 488 /** 489 * Test listener with an {@link OptionClass} specified to be used for arguments testing. 490 */ 491 @OptionClass(alias = "test-alias-class") 492 public static class TestListener extends BaseMetricListener { 493 TestListener(Bundle b)494 public TestListener(Bundle b) { 495 super(b); 496 } 497 } 498 499 /** 500 * Test when the listener does not have an {@link OptionClass} specified, option with alias 501 * are filtered. 502 */ 503 @MetricOption(group = "testGroup") 504 @Test testArgsAlias_noOptionClass()505 public void testArgsAlias_noOptionClass() throws Exception { 506 Bundle args = new Bundle(); 507 args.putString("optionalias:optionname", "optionvalue"); 508 args.putString("noalias", "noaliasvalue"); 509 mListener = createWithArgs(args); 510 mListener.setInstrumentation(mMockInstrumentation); 511 Description runDescription = Description.createSuiteDescription("run"); 512 mListener.testRunStarted(runDescription); 513 Bundle filteredArgs = mListener.getArgsBundle(); 514 assertFalse(filteredArgs.containsKey("optionalias:optionname")); 515 assertFalse(filteredArgs.containsKey("optionname")); 516 assertTrue(filteredArgs.containsKey("noalias")); 517 } 518 519 /** 520 * Test when a listener does have an {@link OptionClass} specified, in that case only options 521 * matching the alias or with no alias are preserved. 522 */ 523 @MetricOption(group = "testGroup") 524 @Test testArgsAlias_optionClass()525 public void testArgsAlias_optionClass() throws Exception { 526 Bundle args = new Bundle(); 527 args.putString("test-alias-class:optionname", "optionvalue"); 528 args.putString("noalias", "noaliasvalue"); 529 args.putString("anotheralias:optionname2", "value"); 530 TestListener listener = new TestListener(args); 531 listener.setInstrumentation(mMockInstrumentation); 532 Description runDescription = Description.createSuiteDescription("run"); 533 listener.testRunStarted(runDescription); 534 Bundle filteredArgs = listener.getArgsBundle(); 535 assertTrue(filteredArgs.containsKey("noalias")); 536 assertTrue(filteredArgs.containsKey("optionname")); 537 538 assertFalse(filteredArgs.containsKey("test-alias-class:optionname")); 539 assertFalse(filteredArgs.containsKey("anotheralias:optionname2")); 540 assertFalse(filteredArgs.containsKey("optionname2")); 541 } 542 543 /** 544 * Report the metrics for the same method name called multiple times. 545 */ 546 @MetricOption(group = "testGroup") 547 @Test testReportMetricsSameMethodName()548 public void testReportMetricsSameMethodName() throws Exception { 549 Description runDescription = Description.createSuiteDescription("run"); 550 mListener.testRunStarted(runDescription); 551 Description testDescription = Description.createTestDescription("class", "method"); 552 mListener.testStarted(testDescription); 553 mListener.testFinished(testDescription); 554 mListener.testStarted(testDescription); 555 mListener.testFinished(testDescription); 556 mListener.testRunFinished(new Result()); 557 // AJUR runner is then gonna call instrumentationRunFinished 558 Bundle resultBundle = new Bundle(); 559 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 560 561 // Check that the in progress status contains the metrics. 562 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 563 Mockito.verify(mMockInstrumentation, Mockito.times(2)) 564 .sendStatus(Mockito.eq( 565 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 566 567 List<Bundle> capturedBundle = capture.getAllValues(); 568 assertEquals(2, capturedBundle.size()); 569 Bundle check = capturedBundle.get(0); 570 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 571 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 572 assertEquals(2, check.size()); 573 574 Bundle check2 = capturedBundle.get(1); 575 assertEquals(TEST_START_VALUE + "method", check2.getString(TEST_START_KEY)); 576 assertEquals(TEST_END_VALUE + "method", check2.getString(TEST_END_KEY)); 577 assertEquals(2, check2.size()); 578 579 // Check that final bundle contains run results 580 assertEquals(RUN_START_VALUE, resultBundle.getString(RUN_START_KEY)); 581 assertEquals(RUN_END_VALUE, resultBundle.getString(RUN_END_KEY)); 582 assertEquals(2, resultBundle.size()); 583 } 584 585 586 /** 587 * Metric collection happens only during given intervals. 588 */ 589 @MetricOption(group = "testGroup") 590 @Test testSameMethodNameWithValidIntervalOption()591 public void testSameMethodNameWithValidIntervalOption() throws Exception { 592 Bundle args = new Bundle(); 593 args.putString(BaseMetricListener.COLLECT_ITERATION_INTERVAL, "2"); 594 mListener = createWithArgs(args); 595 mListener.setInstrumentation(mMockInstrumentation); 596 597 Description runDescription = Description.createSuiteDescription("run"); 598 mListener.testRunStarted(runDescription); 599 Description testDescription = Description.createTestDescription("class", "method"); 600 mListener.testStarted(testDescription); 601 mListener.testFinished(testDescription); 602 mListener.testStarted(testDescription); 603 mListener.testFinished(testDescription); 604 mListener.testStarted(testDescription); 605 mListener.testFinished(testDescription); 606 mListener.testStarted(testDescription); 607 mListener.testFinished(testDescription); 608 mListener.testRunFinished(new Result()); 609 // AJUR runner is then gonna call instrumentationRunFinished 610 Bundle resultBundle = new Bundle(); 611 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 612 613 // Check that the in progress status contains the metrics only for the 2nd and 614 // 4th iteration with the same test name. 615 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 616 Mockito.verify(mMockInstrumentation, Mockito.times(2)) 617 .sendStatus(Mockito.eq( 618 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 619 620 List<Bundle> capturedBundle = capture.getAllValues(); 621 assertEquals(2, capturedBundle.size()); 622 Bundle check = capturedBundle.get(0); 623 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 624 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 625 assertEquals(2, check.size()); 626 627 Bundle check2 = capturedBundle.get(1); 628 assertEquals(TEST_START_VALUE + "method", check2.getString(TEST_START_KEY)); 629 assertEquals(TEST_END_VALUE + "method", check2.getString(TEST_END_KEY)); 630 assertEquals(2, check2.size()); 631 632 // Check that final bundle contains run results 633 assertEquals(RUN_START_VALUE, resultBundle.getString(RUN_START_KEY)); 634 assertEquals(RUN_END_VALUE, resultBundle.getString(RUN_END_KEY)); 635 assertEquals(2, resultBundle.size()); 636 } 637 638 /** 639 * Metric collection does not happen on the skipped iterations. 640 */ 641 @MetricOption(group = "testGroup") 642 @Test testSameMethodNameWithSkipIterationOption()643 public void testSameMethodNameWithSkipIterationOption() throws Exception { 644 Bundle args = new Bundle(); 645 args.putString(BaseMetricListener.SKIP_METRIC_UNTIL_ITERATION, "2"); 646 mListener = createWithArgs(args); 647 mListener.setInstrumentation(mMockInstrumentation); 648 649 // Skip until iteration is set to 2. 650 // Metric will not be collected for 1st and 2nd iterations. 651 Description runDescription = Description.createSuiteDescription("run"); 652 mListener.testRunStarted(runDescription); 653 Description testDescription = Description.createTestDescription("class", "method"); 654 mListener.testStarted(testDescription); 655 mListener.testFinished(testDescription); 656 mListener.testStarted(testDescription); 657 mListener.testFinished(testDescription); 658 mListener.testStarted(testDescription); 659 mListener.testFinished(testDescription); 660 mListener.testStarted(testDescription); 661 mListener.testFinished(testDescription); 662 mListener.testStarted(testDescription); 663 mListener.testFinished(testDescription); 664 mListener.testRunFinished(new Result()); 665 // AJUR runner is then gonna call instrumentationRunFinished 666 Bundle resultBundle = new Bundle(); 667 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 668 669 // Check instrumentation status inprogress called only 3 times. 670 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 671 Mockito.verify(mMockInstrumentation, Mockito.times(3)) 672 .sendStatus(Mockito.eq( 673 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 674 675 List<Bundle> capturedBundle = capture.getAllValues(); 676 assertEquals(3, capturedBundle.size()); 677 Bundle check = capturedBundle.get(0); 678 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 679 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 680 assertEquals(2, check.size()); 681 682 Bundle check2 = capturedBundle.get(1); 683 assertEquals(TEST_START_VALUE + "method", check2.getString(TEST_START_KEY)); 684 assertEquals(TEST_END_VALUE + "method", check2.getString(TEST_END_KEY)); 685 assertEquals(2, check2.size()); 686 687 Bundle check3 = capturedBundle.get(2); 688 assertEquals(TEST_START_VALUE + "method", check3.getString(TEST_START_KEY)); 689 assertEquals(TEST_END_VALUE + "method", check3.getString(TEST_END_KEY)); 690 assertEquals(2, check2.size()); 691 692 // Check that final bundle contains run results 693 assertEquals(RUN_START_VALUE, resultBundle.getString(RUN_START_KEY)); 694 assertEquals(RUN_END_VALUE, resultBundle.getString(RUN_END_KEY)); 695 assertEquals(2, resultBundle.size()); 696 } 697 698 /** 699 * Metric collection happens on all the iteration if the interval is 700 * invalid (i.e less than 1). 701 */ 702 @MetricOption(group = "testGroup") 703 @Test testSameMethodNameWithInvalidIntervalNumber()704 public void testSameMethodNameWithInvalidIntervalNumber() throws Exception { 705 Bundle args = new Bundle(); 706 args.putString(BaseMetricListener.COLLECT_ITERATION_INTERVAL, "0"); 707 mListener = createWithArgs(args); 708 mListener.setInstrumentation(mMockInstrumentation); 709 710 Description runDescription = Description.createSuiteDescription("run"); 711 mListener.testRunStarted(runDescription); 712 Description testDescription = Description.createTestDescription("class", "method"); 713 mListener.testStarted(testDescription); 714 mListener.testFinished(testDescription); 715 mListener.testStarted(testDescription); 716 mListener.testFinished(testDescription); 717 mListener.testRunFinished(new Result()); 718 // AJUR runner is then gonna call instrumentationRunFinished 719 Bundle resultBundle = new Bundle(); 720 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 721 722 // Check that the in progress status contains the metrics only for the 2nd and 723 // 4th iteration with the same test name. 724 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 725 Mockito.verify(mMockInstrumentation, Mockito.times(2)) 726 .sendStatus(Mockito.eq( 727 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 728 729 List<Bundle> capturedBundle = capture.getAllValues(); 730 assertEquals(2, capturedBundle.size()); 731 Bundle check = capturedBundle.get(0); 732 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 733 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 734 assertEquals(2, check.size()); 735 736 Bundle check2 = capturedBundle.get(1); 737 assertEquals(TEST_START_VALUE + "method", check2.getString(TEST_START_KEY)); 738 assertEquals(TEST_END_VALUE + "method", check2.getString(TEST_END_KEY)); 739 assertEquals(2, check2.size()); 740 741 // Check that final bundle contains run results 742 assertEquals(RUN_START_VALUE, resultBundle.getString(RUN_START_KEY)); 743 assertEquals(RUN_END_VALUE, resultBundle.getString(RUN_END_KEY)); 744 assertEquals(2, resultBundle.size()); 745 } 746 747 /** Test that the report as instrumentation result option works. */ 748 @MetricOption(group = "testGroup") 749 @Test testReportAsInstrumentationResultsIfEnabled()750 public void testReportAsInstrumentationResultsIfEnabled() throws Exception { 751 mListener.setReportAsInstrumentationResults(true); 752 753 Description runDescription = Description.createSuiteDescription("run"); 754 mListener.testRunStarted(runDescription); 755 Description testDescription = Description.createTestDescription("class", "method"); 756 mListener.testStarted(testDescription); 757 mListener.testFinished(testDescription); 758 mListener.testRunFinished(new Result()); 759 // AJUR runner is then gonna call instrumentationRunFinished 760 Bundle resultBundle = new Bundle(); 761 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 762 763 // Check that results are reported via Instrumentation.addResults(). 764 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 765 Mockito.verify(mMockInstrumentation, Mockito.times(1)).addResults(capture.capture()); 766 Bundle addedResult = capture.getValue(); 767 assertTrue(addedResult.containsKey(TEST_END_KEY)); 768 assertEquals(TEST_END_VALUE + "method", addedResult.getString(TEST_END_KEY)); 769 770 // Rather than Instrumentation.sendStatus(). 771 Mockito.verify(mMockInstrumentation, Mockito.never()) 772 .sendStatus( 773 Mockito.eq(SendToInstrumentation.INST_STATUS_IN_PROGRESS), 774 Mockito.any(Bundle.class)); 775 } 776 } 777