1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package android.platform.test.rule;
17 
18 import static com.google.common.truth.Truth.assertThat;
19 
20 import android.app.Instrumentation;
21 import android.device.collectors.BaseMetricListener;
22 import android.device.collectors.DataRecord;
23 import android.os.Bundle;
24 
25 import org.junit.Before;
26 import org.junit.Rule;
27 import org.junit.Test;
28 import org.junit.rules.ExpectedException;
29 import org.junit.runner.Description;
30 import org.junit.runner.notification.Failure;
31 import org.junit.runners.model.Statement;
32 
33 import java.util.ArrayList;
34 import java.util.List;
35 
36 /** Tests for {@link TestMetricRule}. */
37 public class TestMetricRuleTest {
38     // A static variable is used so that the metric listeners in this test can modify its state
39     // while being static and able to be created via an empty constructor. This route is taken as
40     // the metric collectors themselves are not directly observable due to being created via
41     // reflection fron TestMetricRule.
42     private static List<String> sLogs = new ArrayList<String>();
43 
44     private static final Description DESCRIPTION =
45             Description.createTestDescription("class", "method");
46 
47     private static final String TEST_FAILURE_MESSAGE = "Failed!";
48 
49     private static final Statement PASSING_STATEMENT =
50             new Statement() {
51                 @Override
52                 public void evaluate() {
53                     sLogs.add("Test execution");
54                 }
55             };
56 
57     private static final Statement FAILING_STATEMENT =
58             new Statement() {
59                 @Override
60                 public void evaluate() {
61                     sLogs.add("Test execution");
62                     throw new RuntimeException(TEST_FAILURE_MESSAGE);
63                 }
64             };
65 
66     @Rule public ExpectedException expectedException = ExpectedException.none();
67 
68     @Before
setUp()69     public void setUp() {
70         sLogs.clear();
71     }
72 
73     @Test
testValidListener_testPasses()74     public void testValidListener_testPasses() throws Throwable {
75         TestMetricRule rule =
76                 createWithMetricCollectorNames(
77                         "android.platform.test.rule.TestMetricRuleTest$TestableCollector1");
78         rule.apply(PASSING_STATEMENT, DESCRIPTION).evaluate();
79         assertThat(sLogs)
80                 .containsExactly(
81                         "TestableCollector1#setInstrumentation",
82                         "TestableCollector1#setupAdditionalArgs",
83                         String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION),
84                         "Test execution",
85                         String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION))
86                 .inOrder();
87     }
88 
89     @Test
testValidListener_testFails()90     public void testValidListener_testFails() throws Throwable {
91         TestMetricRule rule =
92                 createWithMetricCollectorNames(
93                         "android.platform.test.rule.TestMetricRuleTest$TestableCollector1");
94         expectedException.expectMessage(TEST_FAILURE_MESSAGE);
95         rule.apply(FAILING_STATEMENT, DESCRIPTION).evaluate();
96         Failure failure = new Failure(DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE));
97         assertThat(sLogs)
98                 .containsExactly(
99                         "TestableCollector1#setInstrumentation",
100                         "TestableCollector1#setupAdditionalArgs",
101                         String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION),
102                         "Test execution",
103                         String.format(
104                                 "Test %s: TestableCollector1#onTestFail with failure %s",
105                                 DESCRIPTION,
106                                 new Failure(
107                                         DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE))),
108                         String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION))
109                 .inOrder();
110     }
111 
112     @Test
testMultipleListeners_allValid()113     public void testMultipleListeners_allValid() throws Throwable {
114         TestMetricRule rule =
115                 createWithMetricCollectorNames(
116                         "android.platform.test.rule.TestMetricRuleTest$TestableCollector2",
117                         "android.platform.test.rule.TestMetricRuleTest$TestableCollector1");
118         expectedException.expectMessage(TEST_FAILURE_MESSAGE);
119         rule.apply(FAILING_STATEMENT, DESCRIPTION).evaluate();
120         Failure failure = new Failure(DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE));
121         assertThat(sLogs)
122                 .containsExactly(
123                         "TestableCollector1#setInstrumentation",
124                         "TestableCollector1#setupAdditionalArgs",
125                         "TestableCollector2#setInstrumentation",
126                         "TestableCollector2#setupAdditionalArgs",
127                         String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION),
128                         String.format("Test %s: TestableCollector2#onTestStart", DESCRIPTION),
129                         "Test execution",
130                         String.format(
131                                 "Test %s: TestableCollector1#onTestFail with failure %s",
132                                 DESCRIPTION, failure),
133                         String.format(
134                                 "Test %s: TestableCollector2#onTestFail with failure %s",
135                                 DESCRIPTION, failure),
136                         String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION),
137                         String.format("Test %s: TestableCollector2#onTestEnd", DESCRIPTION))
138                 .inOrder();
139     }
140 
141     @Test
testInvalidListenerNameThrows()142     public void testInvalidListenerNameThrows() {
143         String invalidName = "not.a.Collector";
144         expectedException.expectMessage(
145                 String.format(
146                         "Failed to dynamically load metric collector with fully qualified name %s.",
147                         invalidName));
148         // The creation should fail.
149         TestMetricRule rule =
150                 createWithMetricCollectorNames(
151                         "android.platform.test.rule.TestMetricRuleTest$TestableCollector1",
152                         invalidName);
153     }
154 
155     @Test
testSimpleClassNameAttemptedWithKnownPackage()156     public void testSimpleClassNameAttemptedWithKnownPackage() {
157         String simpleName = "NonExistentCollector";
158         // We can't validate real collectors here due to test logistics, so we proxy it by using
159         // an invalid name, and checking that the full name we give in the exception.
160         expectedException.expectMessage(
161                 String.format("%s.%s", TestMetricRule.METRIC_COLLECTORS_PACKAGE, simpleName));
162         // The creation should fail.
163         TestMetricRule rule = createWithMetricCollectorNames(simpleName);
164     }
165 
createWithMetricCollectorNames(String... names)166     private TestMetricRule createWithMetricCollectorNames(String... names) {
167         Bundle args = new Bundle();
168         args.putString(TestMetricRule.METRIC_COLLECTORS_OPTION, String.join(",", names));
169         return new TestMetricRule(args);
170     }
171 
172     public static class BaseTestableCollector extends BaseMetricListener {
173         private final String mName;
174 
BaseTestableCollector(String name)175         public BaseTestableCollector(String name) {
176             mName = name;
177         }
178 
179         @Override
setInstrumentation(Instrumentation instr)180         public void setInstrumentation(Instrumentation instr) {
181             sLogs.add(String.format("%s#%s", mName, "setInstrumentation"));
182         }
183 
184         @Override
setupAdditionalArgs()185         public void setupAdditionalArgs() {
186             sLogs.add(String.format("%s#%s", mName, "setupAdditionalArgs"));
187         }
188 
189         @Override
onTestStart(DataRecord testData, Description description)190         public void onTestStart(DataRecord testData, Description description) {
191             sLogs.add(String.format("Test %s: %s#%s", description, mName, "onTestStart"));
192         }
193 
194         @Override
onTestEnd(DataRecord testData, Description description)195         public void onTestEnd(DataRecord testData, Description description) {
196             sLogs.add(String.format("Test %s: %s#%s", description, mName, "onTestEnd"));
197         }
198 
199         @Override
onTestFail(DataRecord testData, Description description, Failure failure)200         public void onTestFail(DataRecord testData, Description description, Failure failure) {
201             sLogs.add(
202                     String.format(
203                             "Test %s: %s#%s with failure %s",
204                             description, mName, "onTestFail", failure));
205         }
206     }
207 
208     public static class TestableCollector1 extends BaseTestableCollector {
TestableCollector1()209         public TestableCollector1() {
210             super(TestableCollector1.class.getSimpleName());
211         }
212     }
213 
214     public static class TestableCollector2 extends BaseTestableCollector {
TestableCollector2()215         public TestableCollector2() {
216             super(TestableCollector2.class.getSimpleName());
217         }
218     }
219 }
220