1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.metricregression;
17 
18 import com.android.tradefed.log.LogUtil.CLog;
19 import com.android.tradefed.result.TestDescription;
20 import com.android.tradefed.util.MetricsXmlParser;
21 import com.android.tradefed.util.MultiMap;
22 import com.android.tradefed.util.Pair;
23 
24 import com.google.common.annotations.VisibleForTesting;
25 
26 /** A metrics object to hold run metrics and test metrics parsed by {@link MetricsXmlParser} */
27 public class Metrics {
28     private int mNumRuns;
29     private int mNumTests = -1;
30     private final boolean mStrictMode;
31     private final MultiMap<String, Double> mRunMetrics = new MultiMap<>();
32     private final MultiMap<Pair<TestDescription, String>, Double> mTestMetrics = new MultiMap<>();
33 
34     /** Throw when metrics validation fails in strict mode. */
35     public static class MetricsException extends RuntimeException {
MetricsException(String cause)36         MetricsException(String cause) {
37             super(cause);
38         }
39     }
40 
41     /**
42      * Constructs an empty Metrics object.
43      *
44      * @param strictMode whether exception should be thrown when validation fails
45      */
Metrics(boolean strictMode)46     public Metrics(boolean strictMode) {
47         mStrictMode = strictMode;
48     }
49 
50     /**
51      * Sets the number of tests. This method also checks if each call sets the same number of test,
52      * since this number should be consistent across multiple runs.
53      *
54      * @param numTests the number of tests
55      * @throws MetricsException if subsequent calls set a different number.
56      */
setNumTests(int numTests)57     public void setNumTests(int numTests) {
58         if (mNumTests == -1) {
59             mNumTests = numTests;
60         } else {
61             if (mNumTests != numTests) {
62                 String msg =
63                         String.format(
64                                 "Number of test entries differ: expect #%d actual #%d",
65                                 mNumTests, numTests);
66                 throw new MetricsException(msg);
67             }
68         }
69     }
70 
71     /**
72      * Adds a run metric.
73      *
74      * @param name metric name
75      * @param value metric value
76      */
addRunMetric(String name, String value)77     public void addRunMetric(String name, String value) {
78         try {
79             mRunMetrics.put(name, Double.parseDouble(value));
80         } catch (NumberFormatException e) {
81             // This is normal. We often get some string metrics like device name. Just log it.
82             CLog.w(String.format("Run metric \"%s\" is not a number: \"%s\"", name, value));
83         }
84     }
85 
86     /**
87      * Adds a test metric.
88      *
89      * @param testId TestDescription of the metric
90      * @param name metric name
91      * @param value metric value
92      */
addTestMetric(TestDescription testId, String name, String value)93     public void addTestMetric(TestDescription testId, String name, String value) {
94         Pair<TestDescription, String> metricId = new Pair<>(testId, name);
95         try {
96             mTestMetrics.put(metricId, Double.parseDouble(value));
97         } catch (NumberFormatException e) {
98             // This is normal. We often get some string metrics like device name. Just log it.
99             CLog.w(
100                     String.format(
101                             "Test %s metric \"%s\" is not a number: \"%s\"", testId, name, value));
102         }
103     }
104 
105     /**
106      * Validates that the number of entries of each metric equals to the number of runs.
107      *
108      * @param numRuns number of runs
109      * @throws MetricsException when validation fails in strict mode
110      */
validate(int numRuns)111     public void validate(int numRuns) {
112         mNumRuns = numRuns;
113         for (String name : mRunMetrics.keySet()) {
114             if (mRunMetrics.get(name).size() < mNumRuns) {
115                 error(
116                         String.format(
117                                 "Run metric \"%s\" too few entries: expected #%d actual #%d",
118                                 name, mNumRuns, mRunMetrics.get(name).size()));
119             }
120         }
121         for (Pair<TestDescription, String> id : mTestMetrics.keySet()) {
122             if (mTestMetrics.get(id).size() < mNumRuns) {
123                 error(
124                         String.format(
125                                 "Test %s metric \"%s\" too few entries: expected #%d actual #%d",
126                                 id.first, id.second, mNumRuns, mTestMetrics.get(id).size()));
127             }
128         }
129     }
130 
131     /**
132      * Validates with after-patch Metrics object. Make sure two metrics object contain same run
133      * metric entries and test metric entries. Assume this object contains before-patch metrics.
134      *
135      * @param after a Metrics object containing after-patch metrics
136      * @throws MetricsException when cross validation fails in strict mode
137      */
crossValidate(Metrics after)138     public void crossValidate(Metrics after) {
139         if (mNumTests != after.mNumTests) {
140             error(
141                     String.format(
142                             "Number of test entries differ: before #%d after #%d",
143                             mNumTests, after.mNumTests));
144         }
145 
146         for (String name : mRunMetrics.keySet()) {
147             if (!after.mRunMetrics.containsKey(name)) {
148                 warn(String.format("Run metric \"%s\" only in before-patch run.", name));
149             }
150         }
151 
152         for (String name : after.mRunMetrics.keySet()) {
153             if (!mRunMetrics.containsKey(name)) {
154                 warn(String.format("Run metric \"%s\" only in after-patch run.", name));
155             }
156         }
157 
158         for (Pair<TestDescription, String> id : mTestMetrics.keySet()) {
159             if (!after.mTestMetrics.containsKey(id)) {
160                 warn(
161                         String.format(
162                                 "Test %s metric \"%s\" only in before-patch run.",
163                                 id.first, id.second));
164             }
165         }
166 
167         for (Pair<TestDescription, String> id : after.mTestMetrics.keySet()) {
168             if (!mTestMetrics.containsKey(id)) {
169                 warn(
170                         String.format(
171                                 "Test %s metric \"%s\" only in after-patch run.",
172                                 id.first, id.second));
173             }
174         }
175     }
176 
177     @VisibleForTesting
error(String msg)178     void error(String msg) {
179         if (mStrictMode) {
180             throw new MetricsException(msg);
181         } else {
182             CLog.e(msg);
183         }
184     }
185 
186     @VisibleForTesting
warn(String msg)187     void warn(String msg) {
188         if (mStrictMode) {
189             throw new MetricsException(msg);
190         } else {
191             CLog.w(msg);
192         }
193     }
194 
195     /**
196      * Gets the number of test runs stored in this object.
197      *
198      * @return number of test runs
199      */
getNumRuns()200     public int getNumRuns() {
201         return mNumRuns;
202     }
203 
204     /**
205      * Gets the number of tests stored in this object.
206      *
207      * @return number of tests
208      */
getNumTests()209     public int getNumTests() {
210         return mNumTests;
211     }
212 
213     /**
214      * Gets all run metrics stored in this object.
215      *
216      * @return a {@link MultiMap} from test name String to Double
217      */
getRunMetrics()218     public MultiMap<String, Double> getRunMetrics() {
219         return mRunMetrics;
220     }
221 
222     /**
223      * Gets all test metrics stored in this object.
224      *
225      * @return a {@link MultiMap} from (TestDescription, test name) pair to Double
226      */
getTestMetrics()227     public MultiMap<Pair<TestDescription, String>, Double> getTestMetrics() {
228         return mTestMetrics;
229     }
230 }
231