1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.cts.tradefed.result;
17 
18 import static com.android.cts.tradefed.result.CtsXmlResultReporter.CTS_RESULT_FILE_VERSION;
19 
20 import com.android.cts.tradefed.UnitTests;
21 import com.android.cts.util.AbiUtils;
22 import com.android.ddmlib.testrunner.TestIdentifier;
23 import com.android.tradefed.build.IFolderBuildInfo;
24 import com.android.tradefed.log.LogUtil.CLog;
25 import com.android.tradefed.result.LogDataType;
26 import com.android.tradefed.result.LogFile;
27 import com.android.tradefed.result.TestSummary;
28 import com.android.tradefed.result.XmlResultReporter;
29 import com.android.tradefed.util.FileUtil;
30 
31 import junit.framework.TestCase;
32 
33 import org.easymock.EasyMock;
34 
35 import java.io.ByteArrayOutputStream;
36 import java.io.File;
37 import java.io.IOException;
38 import java.io.OutputStream;
39 import java.util.Arrays;
40 import java.util.ArrayList;
41 import java.util.Collections;
42 import java.util.HashMap;
43 import java.util.List;
44 import java.util.Map;
45 
46 /**
47  * Unit tests for {@link XmlResultReporter}.
48  */
49 public class CtsXmlResultReporterTest extends TestCase {
50 
51     private static final List<TestSummary> SUMMARY_LIST =
52             new ArrayList<>(Arrays.asList(new TestSummary("TEST_SUMMARY_URL")));
53     private CtsXmlResultReporter mResultReporter;
54     private ByteArrayOutputStream mOutputStream;
55     private File mBuildDir;
56     private File mReportDir;
57     private IFolderBuildInfo mMockBuild;
58 
59     /**
60      * {@inheritDoc}
61      */
62     @Override
setUp()63     protected void setUp() throws Exception {
64         super.setUp();
65 
66         mOutputStream = new ByteArrayOutputStream();
67         mResultReporter = new CtsXmlResultReporter() {
68             @Override
69             OutputStream createOutputResultStream(File reportDir) throws IOException {
70                 return mOutputStream;
71             }
72 
73             @Override
74             String getTimestamp() {
75                 return "ignore";
76             }
77         };
78         // TODO: use mock file dir instead
79         mReportDir = FileUtil.createTempDir("foo");
80         mResultReporter.setReportDir(mReportDir);
81         mBuildDir = FileUtil.createTempDir("build");
82         File ctsDir = new File(mBuildDir, "android-cts");
83         File repoDir = new File(ctsDir, "repository");
84         File casesDir = new File(repoDir, "testcases");
85         File plansDir = new File(repoDir, "plans");
86         assertTrue(casesDir.mkdirs());
87         assertTrue(plansDir.mkdirs());
88         mMockBuild = EasyMock.createMock(IFolderBuildInfo.class);
89         EasyMock.expect(mMockBuild.getDeviceSerial()).andStubReturn(null);
90         EasyMock.expect(mMockBuild.getRootDir()).andStubReturn(mBuildDir);
91         mMockBuild.addBuildAttribute(EasyMock.cmpEq(CtsXmlResultReporter.CTS_RESULT_DIR),
92                 (String) EasyMock.anyObject());
93         EasyMock.expectLastCall();
94         Map<String, String> attributes = new HashMap<>();
95         attributes.put(CtsXmlResultReporter.CTS_RESULT_DIR, "");
96         EasyMock.expect(mMockBuild.getBuildAttributes()).andStubReturn(attributes);
97         EasyMock.expect(mMockBuild.getBuildId()).andStubReturn("");
98     }
99 
100     @Override
tearDown()101     protected void tearDown() throws Exception {
102         if (mReportDir != null) {
103             FileUtil.recursiveDelete(mReportDir);
104         }
105         if (mBuildDir != null) {
106             FileUtil.recursiveDelete(mBuildDir);
107         }
108         super.tearDown();
109     }
110 
111     /**
112      * A simple test to ensure expected output is generated for test run with no tests.
113      */
testEmptyGeneration()114     public void testEmptyGeneration() {
115         final String expectedHeaderOutput = "<?xml version='1.0' encoding='UTF-8' standalone='no' ?>" +
116             "<?xml-stylesheet type=\"text/xsl\" href=\"cts_result.xsl\"?>";
117         final String expectedTestOutput = String.format(
118             "<TestResult testPlan=\"NA\" starttime=\"ignore\" endtime=\"ignore\" " +
119                     "version=\"%s\" suite=\"%s\"> ", CTS_RESULT_FILE_VERSION, "CTS" );
120         final String expectedSummaryOutput =
121             "<Summary failed=\"0\" notExecuted=\"0\" timeout=\"0\" pass=\"0\" />";
122         final String expectedEndTag = "</TestResult>";
123         EasyMock.replay(mMockBuild);
124         mResultReporter.invocationStarted(mMockBuild);
125         mResultReporter.invocationEnded(1);
126         String actualOutput = getOutput();
127         assertTrue(actualOutput.startsWith(expectedHeaderOutput));
128         assertTrue(String.format("test output did not contain expected test result [%s]. Got %s",
129                 expectedTestOutput, actualOutput), actualOutput.contains(expectedTestOutput));
130         assertTrue(String.format("test output did not contain expected test summary [%s]. Got %s",
131                 expectedTestOutput, actualOutput), actualOutput.contains(expectedSummaryOutput));
132         assertTrue(String.format("test output did not contain expected TestResult end tag. Got %s",
133                 actualOutput), actualOutput.endsWith(expectedEndTag));
134         EasyMock.verify(mMockBuild);
135     }
136 
137     /**
138      * A simple test to ensure expected output is generated for test run with a single passed test.
139      */
testSinglePass()140     public void testSinglePass() {
141         Map<String, String> emptyMap = Collections.emptyMap();
142         final TestIdentifier testId = new TestIdentifier("com.foo.FooTest", "testFoo");
143         EasyMock.replay(mMockBuild);
144         mResultReporter.invocationStarted(mMockBuild);
145         mResultReporter.testRunStarted(AbiUtils.createId(UnitTests.ABI.getName(), "run"), 1);
146         mResultReporter.testStarted(testId);
147         mResultReporter.testEnded(testId, emptyMap);
148         mResultReporter.testRunEnded(3000, emptyMap);
149         mResultReporter.invocationEnded(1);
150         mResultReporter.putSummary(SUMMARY_LIST);
151         String output =  getOutput();
152         // TODO: consider doing xml based compare
153         assertTrue(output.contains(
154               "<Summary failed=\"0\" notExecuted=\"0\" timeout=\"0\" pass=\"1\" />"));
155         assertTrue(output.contains("<TestPackage name=\"\" appPackageName=\"run\" abi=\"" +
156               UnitTests.ABI.getName() + "\" digest=\"\">"));
157         assertTrue(output.contains("<TestCase name=\"FooTest\" priority=\"\">"));
158 
159         final String testCaseTag = String.format(
160                 "<Test name=\"%s\" result=\"pass\"", testId.getTestName());
161         assertTrue(output.contains(testCaseTag));
162         EasyMock.verify(mMockBuild);
163     }
164 
165     /**
166      * A simple test to ensure expected output is generated for test run with a single failed test.
167      */
testSingleFail()168     public void testSingleFail() {
169         Map<String, String> emptyMap = Collections.emptyMap();
170         final TestIdentifier testId = new TestIdentifier("FooTest", "testFoo");
171         final String trace = "this is a trace\nmore trace\nyet more trace";
172         EasyMock.replay(mMockBuild);
173         mResultReporter.invocationStarted(mMockBuild);
174         mResultReporter.testRunStarted(AbiUtils.createId(UnitTests.ABI.getName(), "run"), 1);
175         mResultReporter.testStarted(testId);
176         mResultReporter.testFailed(testId, trace);
177         mResultReporter.testEnded(testId, emptyMap);
178         mResultReporter.testRunEnded(3, emptyMap);
179         mResultReporter.testLogSaved("logcat-foo-bar", LogDataType.TEXT, null,
180                 new LogFile("path", "url"));
181         mResultReporter.invocationEnded(1);
182         String output = getOutput();
183         // TODO: consider doing xml based compare
184         assertTrue(output.contains(
185                 "<Summary failed=\"1\" notExecuted=\"0\" timeout=\"0\" pass=\"0\" />"));
186         final String failureTag =
187                 "<FailedScene message=\"this is a trace&#10;more trace\">     " +
188                 "<StackTrace>this is a tracemore traceyet more trace</StackTrace>";
189         assertTrue(output.contains(failureTag));
190 
191         // Check that no TestLog tags were added, because the flag wasn't enabled.
192         final String testLogTag = String.format("<TestLog type=\"logcat\" url=\"url\" />");
193         assertFalse(output, output.contains(testLogTag));
194         EasyMock.verify(mMockBuild);
195     }
196 
197     /**
198      * Test that flips the include-test-log-tags flag and checks that logs are written to the XML.
199      */
testIncludeTestLogTags()200     public void testIncludeTestLogTags() {
201         Map<String, String> emptyMap = Collections.emptyMap();
202         final TestIdentifier testId = new TestIdentifier("FooTest", "testFoo");
203         final String trace = "this is a trace\nmore trace\nyet more trace";
204 
205         // Include TestLogTags in the XML.
206         mResultReporter.setIncludeTestLogTags(true);
207 
208         EasyMock.replay(mMockBuild);
209         mResultReporter.invocationStarted(mMockBuild);
210         mResultReporter.testRunStarted(AbiUtils.createId(UnitTests.ABI.getName(), "run"), 1);
211         mResultReporter.testStarted(testId);
212         mResultReporter.testFailed(testId, trace);
213         mResultReporter.testEnded(testId, emptyMap);
214         mResultReporter.testRunEnded(3, emptyMap);
215         mResultReporter.testLogSaved("logcat-foo-bar", LogDataType.TEXT, null,
216                 new LogFile("path", "url"));
217         mResultReporter.invocationEnded(1);
218 
219         // Check for TestLog tags because the flag was enabled via setIncludeTestLogTags.
220         final String output = getOutput();
221         final String testLogTag = String.format("<TestLog type=\"logcat\" url=\"url\" />");
222         assertTrue(output, output.contains(testLogTag));
223         EasyMock.verify(mMockBuild);
224     }
225 
testDeviceSetup()226     public void testDeviceSetup() {
227         Map<String, String> emptyMap = Collections.emptyMap();
228         final TestIdentifier testId = new TestIdentifier("android.tests.devicesetup", "TestDeviceSetup");
229         EasyMock.replay(mMockBuild);
230         mResultReporter.invocationStarted(mMockBuild);
231         mResultReporter.testRunStarted(AbiUtils.createId(UnitTests.ABI.getName(), testId.getClassName()), 1);
232         mResultReporter.testStarted(testId);
233         mResultReporter.testEnded(testId, emptyMap);
234         mResultReporter.testRunEnded(3, emptyMap);
235         mResultReporter.invocationEnded(1);
236         String output = getOutput();
237         // TODO: consider doing xml based compare
238         final String deviceSetupTag = "appPackageName=\"android.tests.devicesetup\"";
239         assertFalse(output, output.contains(deviceSetupTag));
240         EasyMock.verify(mMockBuild);
241     }
242 
243     /**
244      * Gets the output produced, stripping it of extraneous whitespace characters.
245      */
getOutput()246     private String getOutput() {
247         String output = mOutputStream.toString();
248         // ignore newlines and tabs whitespace
249         output = output.replaceAll("[\\r\\n\\t]", "");
250         // replace two ws chars with one
251         return output.replaceAll("  ", " ");
252     }
253 }
254