1 /*
2  * Copyright (C) 2021 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package android.platform.test.rule;
17 
18 import android.app.Instrumentation;
19 import android.device.collectors.BaseMetricListener;
20 import android.os.Bundle;
21 import android.util.Log;
22 import androidx.annotation.VisibleForTesting;
23 import androidx.test.InstrumentationRegistry;
24 
25 import com.google.common.collect.Lists;
26 
27 import org.junit.runner.Description;
28 import org.junit.runner.notification.Failure;
29 
30 import java.lang.reflect.Constructor;
31 import java.util.ArrayList;
32 import java.util.Arrays;
33 import java.util.List;
34 
35 /**
36  * A rule that collects test-level metrics using a supplied list of metric collectors.
37  *
38  * <p>The metric collectors are passed in using the "test-metric-collectors" option, and the rule
39  * works by invoking the correct callbacks on them at the corresponding stages of the test
40  * lifecycle. The metric collectors must be subclasses of {@link BaseMetricListener}, and can be
41  * passed in by their fully qualified class name, or simple class name if they are under the {@code
42  * android.device.collectors} package (but not subpackages).
43  *
44  * <p>Multiple metric collectors are supported as comma-separated values, The order they are
45  * triggered follows this example: for {@code -e test-metric-collectors Collector1,Collector2}, the
46  * evaluation order would be {@code Collector1#testStarted()}, {@code Collector2#testStarted()},
47  * {@code @Test}, {@code Collector1#testFinished()}, {@code Collector1#testFinished()}.
48  *
49  * <p>The above FIFO ordering of listeners can be changed to LIFO if the {@code
50  * test-metric-collectors-fifo-order} option is set to {@code false}.
51  *
52  * <p>For {@code Microbenchmark}s, this rule can be dynamically injected either inside or outside
53  * hardcoded rules (see {@code Microbenchmark})'s JavaDoc).
54  *
55  * <p>Exceptions from metric listeners are silently logged. This behavior is in accordance with the
56  * approach taken by {@link BaseMetricListener}.
57  */
58 public class TestMetricRule extends TestWatcher {
59     @VisibleForTesting static final String METRIC_COLLECTORS_OPTION = "test-metric-collectors";
60     @VisibleForTesting static final String FIFO_ORDER_OPTION = "test-metric-collectors-fifo-order";
61     @VisibleForTesting static final String METRIC_COLLECTORS_PACKAGE = "android.device.collectors";
62 
63     protected List<BaseMetricListener> mMetricListeners = new ArrayList<>();
64     // Listeners invoked when finishing or failing a test. Will be a view on mMetricListeners.
65     protected List<BaseMetricListener> mFinishingListeners;
66     protected boolean mFifoOrder = true;
67     private final String mLogTag;
68 
TestMetricRule()69     public TestMetricRule() {
70         this(InstrumentationRegistry.getArguments());
71     }
72 
73     @VisibleForTesting
TestMetricRule(Bundle args)74     TestMetricRule(Bundle args) {
75         this(
76                 args,
77                 InstrumentationRegistry.getInstrumentation(),
78                 METRIC_COLLECTORS_OPTION,
79                 FIFO_ORDER_OPTION,
80                 TestMetricRule.class.getSimpleName());
81     }
82 
83     /**
84      * A constructor that allows subclasses to change out various components used at initialization
85      * time.
86      */
TestMetricRule( Bundle args, Instrumentation instrumentation, String collectorsOptionName, String fifoOrderOptionName, String logTag)87     protected TestMetricRule(
88             Bundle args,
89             Instrumentation instrumentation,
90             String collectorsOptionName,
91             String fifoOrderOptionName,
92             String logTag) {
93         mLogTag = logTag;
94         mFifoOrder =
95                 Boolean.parseBoolean(args.getString(fifoOrderOptionName, String.valueOf(true)));
96         List<String> listenerNames =
97                 Arrays.asList(args.getString(collectorsOptionName, "").split(","));
98         for (String listenerName : listenerNames) {
99             if (listenerName.isEmpty()) {
100                 continue;
101             }
102             BaseMetricListener listener = null;
103             // We could use a regex here, but this is simpler and should work just as well.
104             if (listenerName.contains(".")) {
105                 Log.i(
106                         mLogTag,
107                         String.format(
108                                 "Attempting to dynamically load metric collector with fully "
109                                         + "qualified name %s.",
110                                 listenerName));
111                 try {
112                     listener = loadListenerByFullyQualifiedName(listenerName);
113                 } catch (Exception e) {
114                     throw new IllegalArgumentException(
115                             String.format(
116                                     "Failed to dynamically load metric collector with fully "
117                                             + "qualified name %s.",
118                                     listenerName),
119                             e);
120                 }
121             } else {
122                 String fullName = String.format("%s.%s", METRIC_COLLECTORS_PACKAGE, listenerName);
123                 Log.i(
124                         mLogTag,
125                         String.format(
126                                 "Attempting to dynamically load metric collector with simple class "
127                                         + "name %s (fully qualified name: %s).",
128                                 listenerName, fullName));
129                 try {
130                     listener = loadListenerByFullyQualifiedName(fullName);
131                 } catch (Exception e) {
132                     throw new IllegalArgumentException(
133                             String.format(
134                                     "Failed to dynamically load metric collector with simple class "
135                                             + "name %s (attempted fully qualified name: %s).",
136                                     listenerName, fullName),
137                             e);
138                 }
139             }
140             mMetricListeners.add(listener);
141         }
142         // Initialize each listener.
143         for (BaseMetricListener listener : mMetricListeners) {
144             listener.setInstrumentation(instrumentation);
145         }
146         mFinishingListeners = mFifoOrder ? mMetricListeners : Lists.reverse(mMetricListeners);
147     }
148 
149     @Override
starting(Description description)150     protected void starting(Description description) {
151         for (BaseMetricListener listener : mMetricListeners) {
152             listener.setUp();
153         }
154         for (BaseMetricListener listener : mMetricListeners) {
155             try {
156                 listener.testStarted(description);
157             } catch (Exception e) {
158                 Log.e(
159                         mLogTag,
160                         String.format(
161                                 "Exception from listener %s during starting().",
162                                 listener.getClass().getCanonicalName()),
163                         e);
164             }
165         }
166     }
167 
168     @Override
finished(Description description)169     protected void finished(Description description) {
170         for (BaseMetricListener listener : mFinishingListeners) {
171             try {
172                 listener.testFinished(description);
173             } catch (Exception e) {
174                 Log.e(
175                         mLogTag,
176                         String.format(
177                                 "Exception from listener %s during finished().",
178                                 listener.getClass().getCanonicalName()),
179                         e);
180             }
181         }
182         for (BaseMetricListener listener : mFinishingListeners) {
183             listener.cleanUp();
184         }
185     }
186 
187     @Override
failed(Throwable t, Description description)188     protected void failed(Throwable t, Description description) {
189         Failure failure = new Failure(description, t);
190         for (BaseMetricListener listener : mFinishingListeners) {
191             try {
192                 listener.testFailure(failure);
193             } catch (Exception e) {
194                 Log.e(
195                         mLogTag,
196                         String.format(
197                                 "Exception from listener %s during failed().",
198                                 listener.getClass().getCanonicalName()),
199                         e);
200             }
201         }
202     }
203 
loadListenerByFullyQualifiedName(String name)204     private BaseMetricListener loadListenerByFullyQualifiedName(String name) throws Exception {
205         // Load the metric collector class using reflection.
206         Class<?> loadedClass = null;
207         try {
208             loadedClass = TestMetricRule.class.getClassLoader().loadClass(name);
209         } catch (ClassNotFoundException e) {
210             throw new IllegalArgumentException(
211                     String.format("Could not find class with fully qualified name %s.", name));
212         }
213         // Ensure that the class found is a BaseMetricListener.
214         if (loadedClass == null || (!BaseMetricListener.class.isAssignableFrom(loadedClass))) {
215             throw new IllegalArgumentException(
216                     String.format("Class %s is not a BaseMetricListener.", loadedClass));
217         }
218         // Use the default constructor to create a metric collector instance.
219         try {
220             Constructor<?> constructor = loadedClass.getConstructor();
221             // Cast is safe as we have vetted that loadedClass is a BaseMetricListener.
222             return (BaseMetricListener) constructor.newInstance();
223         } catch (NoSuchMethodException e) {
224             throw new IllegalArgumentException(
225                     String.format(
226                             "Metric collector %s cannot be instantiated with an empty constructor",
227                             loadedClass),
228                     e);
229         }
230     }
231 }
232