1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package android.platform.test.rule;
17 
18 import android.device.collectors.BaseMetricListener;
19 import android.os.Bundle;
20 import android.util.Log;
21 import androidx.annotation.VisibleForTesting;
22 import androidx.test.InstrumentationRegistry;
23 
24 import org.junit.runner.Description;
25 import org.junit.runner.notification.Failure;
26 
27 import java.lang.reflect.Constructor;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.List;
31 
32 /**
33  * A rule that collects test-level metrics using a supplied list of metric collectors.
34  *
35  * <p>The metric collectors are passed in using the "test-metric-collectors" option, and the rule
36  * works by invoking the correct callbacks on them at the corresponding stages of the test
37  * lifecycle. The metric collectors must be subclasses of {@link BaseMetricListener}, and can be
38  * passed in by their fully qualified class name, or simple class name if they are under the {@code
39  * android.device.collectors} package (but not subpackages).
40  *
41  * <p>Multiple metric collectors are supported as comma-separated values, The order they are
42  * triggered follows this example: for {@code -e test-metric-collectors Collector1,Collector2}, the
43  * evaluation order would be {@code Collector1#testStarted()}, {@code Collector2#testStarted()},
44  * {@code @Test}, {@code Collector1#testFinished()}, {@code Collector1#testFinished()}.
45  *
46  * <p>For {@code Microbenchmark}s, this rule can be dynamically injected either inside or outside
47  * hardcoded rules (see {@code Microbenchmark})'s JavaDoc).
48  *
49  * <p>Exceptions from metric listeners are silently logged. This behavior is in accordance with the
50  * approach taken by {@link BaseMetricListener}.
51  */
52 public class TestMetricRule extends TestWatcher {
53     private static final String LOG_TAG = TestMetricRule.class.getSimpleName();
54 
55     @VisibleForTesting static final String METRIC_COLLECTORS_OPTION = "test-metric-collectors";
56     @VisibleForTesting static final String METRIC_COLLECTORS_PACKAGE = "android.device.collectors";
57 
58     private List<BaseMetricListener> mMetricListeners = new ArrayList<>();
59 
TestMetricRule()60     public TestMetricRule() {
61         this(InstrumentationRegistry.getArguments());
62     }
63 
64     @VisibleForTesting
TestMetricRule(Bundle args)65     TestMetricRule(Bundle args) {
66         List<String> listenerNames =
67                 Arrays.asList(args.getString(METRIC_COLLECTORS_OPTION, "").split(","));
68         for (String listenerName : listenerNames) {
69             if (listenerName.isEmpty()) {
70                 continue;
71             }
72             BaseMetricListener listener = null;
73             // We could use a regex here, but this is simpler and should work just as well.
74             if (listenerName.contains(".")) {
75                 Log.i(
76                         LOG_TAG,
77                         String.format(
78                                 "Attempting to dynamically load metric collector with fully "
79                                         + "qualified name %s.",
80                                 listenerName));
81                 try {
82                     listener = loadListenerByFullyQualifiedName(listenerName);
83                 } catch (Exception e) {
84                     throw new IllegalArgumentException(
85                             String.format(
86                                     "Failed to dynamically load metric collector with fully "
87                                             + "qualified name %s.",
88                                     listenerName),
89                             e);
90                 }
91             } else {
92                 String fullName = String.format("%s.%s", METRIC_COLLECTORS_PACKAGE, listenerName);
93                 Log.i(
94                         LOG_TAG,
95                         String.format(
96                                 "Attempting to dynamically load metric collector with simple class "
97                                         + "name %s (fully qualified name: %s).",
98                                 listenerName, fullName));
99                 try {
100                     listener = loadListenerByFullyQualifiedName(fullName);
101                 } catch (Exception e) {
102                     throw new IllegalArgumentException(
103                             String.format(
104                                     "Failed to dynamically load metric collector with simple class "
105                                             + "name %s (attempted fully qualified name: %s).",
106                                     listenerName, fullName),
107                             e);
108                 }
109             }
110             mMetricListeners.add(listener);
111         }
112         // Initialize each listener.
113         for (BaseMetricListener listener : mMetricListeners) {
114             listener.setInstrumentation(InstrumentationRegistry.getInstrumentation());
115             listener.setupAdditionalArgs();
116         }
117     }
118 
119     @Override
starting(Description description)120     protected void starting(Description description) {
121         for (BaseMetricListener listener : mMetricListeners) {
122             try {
123                 listener.testStarted(description);
124             } catch (Exception e) {
125                 Log.e(
126                         LOG_TAG,
127                         String.format(
128                                 "Exception from listener %s during starting().",
129                                 listener.getClass().getCanonicalName()),
130                         e);
131             }
132         }
133     }
134 
135     @Override
finished(Description description)136     protected void finished(Description description) {
137         for (BaseMetricListener listener : mMetricListeners) {
138             try {
139                 listener.testFinished(description);
140             } catch (Exception e) {
141                 Log.e(
142                         LOG_TAG,
143                         String.format(
144                                 "Exception from listener %s during finished().",
145                                 listener.getClass().getCanonicalName()),
146                         e);
147             }
148         }
149     }
150 
151     @Override
failed(Throwable t, Description description)152     protected void failed(Throwable t, Description description) {
153         Failure failure = new Failure(description, t);
154         for (BaseMetricListener listener : mMetricListeners) {
155             try {
156                 listener.testFailure(failure);
157             } catch (Exception e) {
158                 Log.e(
159                         LOG_TAG,
160                         String.format(
161                                 "Exception from listener %s during failed().",
162                                 listener.getClass().getCanonicalName()),
163                         e);
164             }
165         }
166     }
167 
loadListenerByFullyQualifiedName(String name)168     private BaseMetricListener loadListenerByFullyQualifiedName(String name) throws Exception {
169         // Load the metric collector class using reflection.
170         Class<?> loadedClass = null;
171         try {
172             loadedClass = TestMetricRule.class.getClassLoader().loadClass(name);
173         } catch (ClassNotFoundException e) {
174             throw new IllegalArgumentException(
175                     String.format("Could not find class with fully qualified name %s.", name));
176         }
177         // Ensure that the class found is a BaseMetricListener.
178         if (loadedClass == null || (!BaseMetricListener.class.isAssignableFrom(loadedClass))) {
179             throw new IllegalArgumentException(
180                     String.format("Class %s is not a BaseMetricListener.", loadedClass));
181         }
182         // Use the default constructor to create a metric collector instance.
183         try {
184             Constructor<?> constructor = loadedClass.getConstructor();
185             // Cast is safe as we have vetted that loadedClass is a BaseMetricListener.
186             return (BaseMetricListener) constructor.newInstance();
187         } catch (NoSuchMethodException e) {
188             throw new IllegalArgumentException(
189                     String.format(
190                             "Metric collector %s cannot be instantiated with an empty constructor",
191                             loadedClass),
192                     e);
193         }
194     }
195 }
196