1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.platform.test.rule; 17 18 import static com.google.common.truth.Truth.assertThat; 19 20 import android.app.Instrumentation; 21 import android.device.collectors.BaseMetricListener; 22 import android.device.collectors.DataRecord; 23 import android.os.Bundle; 24 25 import org.junit.Before; 26 import org.junit.Rule; 27 import org.junit.Test; 28 import org.junit.rules.ExpectedException; 29 import org.junit.runner.Description; 30 import org.junit.runner.notification.Failure; 31 import org.junit.runners.model.Statement; 32 33 import java.util.ArrayList; 34 import java.util.List; 35 36 /** Tests for {@link TestMetricRule}. */ 37 public class TestMetricRuleTest { 38 // A static variable is used so that the metric listeners in this test can modify its state 39 // while being static and able to be created via an empty constructor. This route is taken as 40 // the metric collectors themselves are not directly observable due to being created via 41 // reflection fron TestMetricRule. 42 private static List<String> sLogs = new ArrayList<String>(); 43 44 private static final Description DESCRIPTION = 45 Description.createTestDescription("class", "method"); 46 47 private static final String TEST_FAILURE_MESSAGE = "Failed!"; 48 49 private static final Statement PASSING_STATEMENT = 50 new Statement() { 51 @Override 52 public void evaluate() { 53 sLogs.add("Test execution"); 54 } 55 }; 56 57 private static final Statement FAILING_STATEMENT = 58 new Statement() { 59 @Override 60 public void evaluate() { 61 sLogs.add("Test execution"); 62 throw new RuntimeException(TEST_FAILURE_MESSAGE); 63 } 64 }; 65 66 @Rule public ExpectedException expectedException = ExpectedException.none(); 67 68 @Before setUp()69 public void setUp() { 70 sLogs.clear(); 71 } 72 73 @Test testValidListener_testPasses()74 public void testValidListener_testPasses() throws Throwable { 75 TestMetricRule rule = 76 createWithMetricCollectorNames( 77 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1"); 78 rule.apply(PASSING_STATEMENT, DESCRIPTION).evaluate(); 79 assertThat(sLogs) 80 .containsExactly( 81 "TestableCollector1#setInstrumentation", 82 "TestableCollector1#setupAdditionalArgs", 83 "TestableCollector1#onSetUp", 84 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 85 "Test execution", 86 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 87 "TestableCollector1#onCleanUp") 88 .inOrder(); 89 } 90 91 @Test testValidListener_testFails()92 public void testValidListener_testFails() throws Throwable { 93 TestMetricRule rule = 94 createWithMetricCollectorNames( 95 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1"); 96 expectedException.expectMessage(TEST_FAILURE_MESSAGE); 97 rule.apply(FAILING_STATEMENT, DESCRIPTION).evaluate(); 98 Failure failure = new Failure(DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE)); 99 assertThat(sLogs) 100 .containsExactly( 101 "TestableCollector1#setInstrumentation", 102 "TestableCollector1#setupAdditionalArgs", 103 "TestableCollector1#onSetUp", 104 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 105 "Test execution", 106 String.format( 107 "Test %s: TestableCollector1#onTestFail with failure %s", 108 DESCRIPTION, 109 new Failure( 110 DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE))), 111 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 112 "TestableCollector1#onCleanUp") 113 .inOrder(); 114 } 115 116 @Test testMultipleListeners_allValid()117 public void testMultipleListeners_allValid() throws Throwable { 118 TestMetricRule rule = 119 createWithMetricCollectorNames( 120 "android.platform.test.rule.TestMetricRuleTest$TestableCollector2", 121 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1"); 122 expectedException.expectMessage(TEST_FAILURE_MESSAGE); 123 rule.apply(FAILING_STATEMENT, DESCRIPTION).evaluate(); 124 Failure failure = new Failure(DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE)); 125 assertThat(sLogs) 126 .containsExactly( 127 "TestableCollector1#setInstrumentation", 128 "TestableCollector2#setInstrumentation", 129 "TestableCollector1#setupAdditionalArgs", 130 "TestableCollector1#onSetUp", 131 "TestableCollector2#setupAdditionalArgs", 132 "TestableCollector2#onSetUp", 133 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 134 String.format("Test %s: TestableCollector2#onTestStart", DESCRIPTION), 135 "Test execution", 136 String.format( 137 "Test %s: TestableCollector1#onTestFail with failure %s", 138 DESCRIPTION, failure), 139 String.format( 140 "Test %s: TestableCollector2#onTestFail with failure %s", 141 DESCRIPTION, failure), 142 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 143 String.format("Test %s: TestableCollector2#onTestEnd", DESCRIPTION), 144 "TestableCollector1#onCleanUp", 145 "TestableCollector2#onCleanUp") 146 .inOrder(); 147 } 148 149 @Test testMultipleListeners_lifoOrder()150 public void testMultipleListeners_lifoOrder() throws Throwable { 151 Bundle args = new Bundle(); 152 args.putString( 153 TestMetricRule.METRIC_COLLECTORS_OPTION, 154 String.join( 155 ",", 156 new String[] { 157 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1", 158 "android.platform.test.rule.TestMetricRuleTest$TestableCollector2" 159 })); 160 args.putString(TestMetricRule.FIFO_ORDER_OPTION, "false"); 161 TestMetricRule rule = new TestMetricRule(args); 162 expectedException.expectMessage(TEST_FAILURE_MESSAGE); 163 rule.apply(FAILING_STATEMENT, DESCRIPTION).evaluate(); 164 Failure failure = new Failure(DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE)); 165 assertThat(sLogs) 166 .containsExactly( 167 "TestableCollector1#setInstrumentation", 168 "TestableCollector2#setInstrumentation", 169 "TestableCollector1#setupAdditionalArgs", 170 "TestableCollector1#onSetUp", 171 "TestableCollector2#setupAdditionalArgs", 172 "TestableCollector2#onSetUp", 173 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 174 String.format("Test %s: TestableCollector2#onTestStart", DESCRIPTION), 175 "Test execution", 176 String.format( 177 "Test %s: TestableCollector2#onTestFail with failure %s", 178 DESCRIPTION, failure), 179 String.format( 180 "Test %s: TestableCollector1#onTestFail with failure %s", 181 DESCRIPTION, failure), 182 String.format("Test %s: TestableCollector2#onTestEnd", DESCRIPTION), 183 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 184 "TestableCollector2#onCleanUp", 185 "TestableCollector1#onCleanUp") 186 .inOrder(); 187 } 188 189 @Test testInvalidListenerNameThrows()190 public void testInvalidListenerNameThrows() { 191 String invalidName = "not.a.Collector"; 192 expectedException.expectMessage( 193 String.format( 194 "Failed to dynamically load metric collector with fully qualified name %s.", 195 invalidName)); 196 // The creation should fail. 197 TestMetricRule rule = 198 createWithMetricCollectorNames( 199 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1", 200 invalidName); 201 } 202 203 @Test testSimpleClassNameAttemptedWithKnownPackage()204 public void testSimpleClassNameAttemptedWithKnownPackage() { 205 String simpleName = "NonExistentCollector"; 206 // We can't validate real collectors here due to test logistics, so we proxy it by using 207 // an invalid name, and checking that the full name we give in the exception. 208 expectedException.expectMessage( 209 String.format("%s.%s", TestMetricRule.METRIC_COLLECTORS_PACKAGE, simpleName)); 210 // The creation should fail. 211 TestMetricRule rule = createWithMetricCollectorNames(simpleName); 212 } 213 214 @Test testInitWithDifferentOptionNames()215 public void testInitWithDifferentOptionNames() throws Throwable { 216 String listenerOptionName = "another-" + TestMetricRule.METRIC_COLLECTORS_OPTION; 217 String fifoOptionName = "another-" + TestMetricRule.FIFO_ORDER_OPTION; 218 219 Bundle args = new Bundle(); 220 args.putString( 221 listenerOptionName, 222 String.join( 223 ",", 224 new String[] { 225 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1", 226 "android.platform.test.rule.TestMetricRuleTest$TestableCollector2" 227 })); 228 args.putString(fifoOptionName, "false"); 229 TestMetricRule rule = 230 new TestMetricRule( 231 args, new Instrumentation(), listenerOptionName, fifoOptionName, "log tag"); 232 233 rule.apply(PASSING_STATEMENT, DESCRIPTION).evaluate(); 234 assertThat(sLogs) 235 .containsExactly( 236 "TestableCollector1#setInstrumentation", 237 "TestableCollector2#setInstrumentation", 238 "TestableCollector1#setupAdditionalArgs", 239 "TestableCollector1#onSetUp", 240 "TestableCollector2#setupAdditionalArgs", 241 "TestableCollector2#onSetUp", 242 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 243 String.format("Test %s: TestableCollector2#onTestStart", DESCRIPTION), 244 "Test execution", 245 String.format("Test %s: TestableCollector2#onTestEnd", DESCRIPTION), 246 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 247 "TestableCollector2#onCleanUp", 248 "TestableCollector1#onCleanUp") 249 .inOrder(); 250 } 251 createWithMetricCollectorNames(String... names)252 private TestMetricRule createWithMetricCollectorNames(String... names) { 253 Bundle args = new Bundle(); 254 args.putString(TestMetricRule.METRIC_COLLECTORS_OPTION, String.join(",", names)); 255 return new TestMetricRule(args); 256 } 257 258 public static class BaseTestableCollector extends BaseMetricListener { 259 private final String mName; 260 BaseTestableCollector(String name)261 public BaseTestableCollector(String name) { 262 mName = name; 263 } 264 265 @Override setInstrumentation(Instrumentation instr)266 public void setInstrumentation(Instrumentation instr) { 267 sLogs.add(String.format("%s#%s", mName, "setInstrumentation")); 268 } 269 270 @Override setupAdditionalArgs()271 public void setupAdditionalArgs() { 272 sLogs.add(String.format("%s#%s", mName, "setupAdditionalArgs")); 273 } 274 275 @Override onSetUp()276 public void onSetUp() { 277 sLogs.add(String.format("%s#%s", mName, "onSetUp")); 278 } 279 280 @Override onCleanUp()281 public void onCleanUp() { 282 sLogs.add(String.format("%s#%s", mName, "onCleanUp")); 283 } 284 285 @Override onTestStart(DataRecord testData, Description description)286 public void onTestStart(DataRecord testData, Description description) { 287 sLogs.add(String.format("Test %s: %s#%s", description, mName, "onTestStart")); 288 } 289 290 @Override onTestEnd(DataRecord testData, Description description)291 public void onTestEnd(DataRecord testData, Description description) { 292 sLogs.add(String.format("Test %s: %s#%s", description, mName, "onTestEnd")); 293 } 294 295 @Override onTestFail(DataRecord testData, Description description, Failure failure)296 public void onTestFail(DataRecord testData, Description description, Failure failure) { 297 sLogs.add( 298 String.format( 299 "Test %s: %s#%s with failure %s", 300 description, mName, "onTestFail", failure)); 301 } 302 } 303 304 public static class TestableCollector1 extends BaseTestableCollector { TestableCollector1()305 public TestableCollector1() { 306 super(TestableCollector1.class.getSimpleName()); 307 } 308 } 309 310 public static class TestableCollector2 extends BaseTestableCollector { TestableCollector2()311 public TestableCollector2() { 312 super(TestableCollector2.class.getSimpleName()); 313 } 314 } 315 } 316