1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.cts.tradefed.result;
17 
18 import com.android.cts.tradefed.testtype.CtsTest;
19 import com.android.cts.tradefed.util.CtsHostStore;
20 import com.android.cts.util.AbiUtils;
21 import com.android.ddmlib.testrunner.TestIdentifier;
22 import com.android.tradefed.log.LogUtil.CLog;
23 
24 import org.kxml2.io.KXmlSerializer;
25 import org.xmlpull.v1.XmlPullParser;
26 import org.xmlpull.v1.XmlPullParserException;
27 
28 import java.io.IOException;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.Deque;
32 import java.util.HashMap;
33 import java.util.LinkedList;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.regex.Matcher;
37 import java.util.regex.Pattern;
38 
39 /**
40  * Data structure for a CTS test package result.
41  * <p/>
42  * Provides methods to serialize to XML.
43  */
44 class TestPackageResult extends AbstractXmlPullParser {
45 
46     static final String TAG = "TestPackage";
47 
48     public static final String CTS_RESULT_KEY = "CTS_TEST_RESULT";
49 
50     private static final String DIGEST_ATTR = "digest";
51     private static final String APP_PACKAGE_NAME_ATTR = "appPackageName";
52     private static final String NAME_ATTR = "name";
53     private static final String ABI_ATTR = "abi";
54     private static final String ns = CtsXmlResultReporter.ns;
55     private static final String SIGNATURE_TEST_PKG = "android.tests.sigtest";
56 
57     private static final Pattern mCtsLogPattern = Pattern.compile("(.*)\\+\\+\\+\\+(.*)");
58 
59     private String mDeviceSerial;
60     private String mAppPackageName;
61     private String mName;
62     private String mAbi;
63     private String mDigest;
64 
65     private Map<String, String> mMetrics = new HashMap<String, String>();
66     private Map<TestIdentifier, Map<String, String>> mTestMetrics = new HashMap<TestIdentifier, Map<String, String>>();
67 
68     private TestSuite mSuiteRoot = new TestSuite(null);
69 
setDeviceSerial(String deviceSerial)70     public void setDeviceSerial(String deviceSerial) {
71         mDeviceSerial = deviceSerial;
72     }
73 
getDeviceSerial()74     public String getDeviceSerial() {
75         return mDeviceSerial;
76     }
77 
getId()78     public String getId() {
79         return AbiUtils.createId(getAbi(), getAppPackageName());
80     }
81 
setAppPackageName(String appPackageName)82     public void setAppPackageName(String appPackageName) {
83         mAppPackageName = appPackageName;
84     }
85 
getAppPackageName()86     public String getAppPackageName() {
87         return mAppPackageName;
88     }
89 
setName(String name)90     public void setName(String name) {
91         mName = name;
92     }
93 
getName()94     public String getName() {
95         return mName;
96     }
97 
setAbi(String abi)98     public void setAbi(String abi) {
99         mAbi = abi;
100     }
101 
getAbi()102     public String getAbi() {
103         return mAbi;
104     }
105 
setDigest(String digest)106     public void setDigest(String digest) {
107         mDigest = digest;
108     }
109 
getDigest()110     public String getDigest() {
111         return mDigest;
112     }
113 
114     /**
115      * Return the {@link TestSuite}s
116      */
getTestSuites()117     public Collection<TestSuite> getTestSuites() {
118         return mSuiteRoot.getTestSuites();
119     }
120 
121     /**
122      * Adds a test result to this test package
123      *
124      * @param testId
125      */
insertTest(TestIdentifier testId)126     public Test insertTest(TestIdentifier testId) {
127         return findTest(testId, true);
128     }
129 
findTest(TestIdentifier testId, boolean insertIfMissing)130     private Test findTest(TestIdentifier testId, boolean insertIfMissing) {
131         List<String> classNameSegments = new LinkedList<String>();
132         Collections.addAll(classNameSegments, testId.getClassName().split("\\."));
133         if (classNameSegments.size() <= 0) {
134             CLog.e("Unrecognized package name format for test class '%s'",
135                     testId.getClassName());
136             // should never happen
137             classNameSegments.add("UnknownTestClass");
138         }
139         String testCaseName = classNameSegments.remove(classNameSegments.size() - 1);
140         return mSuiteRoot.findTest(classNameSegments, testCaseName, testId.getTestName(), insertIfMissing);
141     }
142 
143 
144     /**
145      * Find the test result for given {@link TestIdentifier}.
146      * @param testId
147      * @return the {@link Test} or <code>null</code>
148      */
findTest(TestIdentifier testId)149     public Test findTest(TestIdentifier testId) {
150         return findTest(testId, false);
151     }
152 
153     /**
154      * Serialize this object and all its contents to XML.
155      *
156      * @param serializer
157      * @throws IOException
158      */
serialize(KXmlSerializer serializer)159     public void serialize(KXmlSerializer serializer) throws IOException {
160         serializer.startTag(ns, TAG);
161         serializeAttribute(serializer, NAME_ATTR, mName);
162         serializeAttribute(serializer, APP_PACKAGE_NAME_ATTR, mAppPackageName);
163         serializeAttribute(serializer, ABI_ATTR, mAbi);
164         serializeAttribute(serializer, DIGEST_ATTR, getDigest());
165         if (SIGNATURE_TEST_PKG.equals(mName)) {
166             serializer.attribute(ns, "signatureCheck", "true");
167         }
168         mSuiteRoot.serialize(serializer);
169         serializer.endTag(ns, TAG);
170     }
171 
172     /**
173      * Helper method to serialize attributes.
174      * Can handle null values. Useful for cases where test package has not been fully populated
175      * such as when unit testing.
176      *
177      * @param attrName
178      * @param attrValue
179      * @throws IOException
180      */
serializeAttribute(KXmlSerializer serializer, String attrName, String attrValue)181     private void serializeAttribute(KXmlSerializer serializer, String attrName, String attrValue)
182             throws IOException {
183         attrValue = attrValue == null ? "" : attrValue;
184         serializer.attribute(ns, attrName, attrValue);
185     }
186 
187     /**
188      * Populates this class with package result data parsed from XML.
189      *
190      * @param parser the {@link XmlPullParser}. Expected to be pointing at start
191      *            of TestPackage tag
192      */
193     @Override
parse(XmlPullParser parser)194     void parse(XmlPullParser parser) throws XmlPullParserException, IOException {
195         if (!parser.getName().equals(TAG)) {
196             throw new XmlPullParserException(String.format(
197                     "invalid XML: Expected %s tag but received %s", TAG, parser.getName()));
198         }
199         setAppPackageName(getAttribute(parser, APP_PACKAGE_NAME_ATTR));
200         setName(getAttribute(parser, NAME_ATTR));
201         setAbi(getAttribute(parser, ABI_ATTR));
202         setDigest(getAttribute(parser, DIGEST_ATTR));
203         int eventType = parser.getEventType();
204         while (eventType != XmlPullParser.END_DOCUMENT) {
205             if (eventType == XmlPullParser.START_TAG && parser.getName().equals(TestSuite.TAG)) {
206                 TestSuite suite = new TestSuite();
207                 suite.parse(parser);
208                 mSuiteRoot.insertSuite(suite);
209             }
210             if (eventType == XmlPullParser.END_TAG && parser.getName().equals(TAG)) {
211                 return;
212             }
213             eventType = parser.next();
214         }
215     }
216 
217     /**
218      * Return a list of {@link TestIdentifier}s contained in this result with the given status
219      *
220      * @param resultFilter the {@link CtsTestStatus} to filter by
221      * @return a collection of {@link TestIdentifier}s
222      */
getTestsWithStatus(CtsTestStatus resultFilter)223     public Collection<TestIdentifier> getTestsWithStatus(CtsTestStatus resultFilter) {
224         Collection<TestIdentifier> tests = new LinkedList<TestIdentifier>();
225         Deque<String> suiteNames = new LinkedList<String>();
226         mSuiteRoot.addTestsWithStatus(tests, suiteNames, resultFilter);
227         return tests;
228     }
229 
230     /**
231      * Populate values in this package result from run metrics
232      * @param metrics A map of metrics from the completed test run.
233      */
populateMetrics(Map<String, String> metrics)234     public void populateMetrics(Map<String, String> metrics) {
235         String name = metrics.get(CtsTest.PACKAGE_NAME_METRIC);
236         if (name != null) {
237             setName(name);
238         }
239         String abi = metrics.get(CtsTest.PACKAGE_ABI_METRIC);
240         if (abi != null) {
241             setAbi(abi);
242         }
243         String digest = metrics.get(CtsTest.PACKAGE_DIGEST_METRIC);
244         if (digest != null) {
245             setDigest(digest);
246         }
247         mMetrics.putAll(metrics);
248 
249         // Collect performance results
250         for (TestIdentifier test : mTestMetrics.keySet()) {
251             // device test can have performance results in test metrics
252             String perfResult = mTestMetrics.get(test).get(CTS_RESULT_KEY);
253             // host test should be checked in CtsHostStore.
254             if (perfResult == null) {
255                 perfResult = CtsHostStore.removeCtsResult(mDeviceSerial, mAbi, test.toString());
256             }
257             if (perfResult != null) {
258                 // CTS result is passed in Summary++++Details format.
259                 // Extract Summary and Details, and pass them.
260                 Matcher m = mCtsLogPattern.matcher(perfResult);
261                 if (m.find()) {
262                     Test result = findTest(test);
263                     result.setResultStatus(CtsTestStatus.PASS);
264                     result.setSummary(m.group(1));
265                     result.setDetails(m.group(2));
266                 } else {
267                     CLog.e("CTS Result unrecognizable:" + perfResult);
268                 }
269             }
270         }
271     }
272 
273     /**
274      * Report the given test as a failure.
275      *
276      * @param test
277      * @param status
278      * @param trace
279      */
reportTestFailure(TestIdentifier test, CtsTestStatus status, String trace)280     public void reportTestFailure(TestIdentifier test, CtsTestStatus status, String trace) {
281         Test result = findTest(test);
282         result.setResultStatus(status);
283         result.setStackTrace(trace);
284     }
285 
286     /**
287      * Report that the given test has completed.
288      *
289      * @param test The {@link TestIdentifier} of the completed test.
290      * @param testMetrics A map holding metrics about the completed test, if any.
291      */
reportTestEnded(TestIdentifier test, Map<String, String> testMetrics)292     public void reportTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
293         Test result = findTest(test);
294         if (!result.getResult().equals(CtsTestStatus.FAIL)) {
295             result.setResultStatus(CtsTestStatus.PASS);
296         }
297         result.updateEndTime();
298         if (mTestMetrics.containsKey(test)) {
299             CLog.e("Test metrics already contains key: " + test);
300         }
301         mTestMetrics.put(test, testMetrics);
302         CLog.i("Test metrics:" + testMetrics);
303     }
304 
305     /**
306      * Return the number of tests with given status
307      *
308      * @param status
309      * @return the total number of tests with given status
310      */
countTests(CtsTestStatus status)311     public int countTests(CtsTestStatus status) {
312         return mSuiteRoot.countTests(status);
313     }
314 
315     /**
316      * @return A map holding the metrics from the test run.
317      */
getMetrics()318     public Map<String, String> getMetrics() {
319         return mMetrics;
320     }
321 
322 }
323