1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.regression.tests; 18 19 import static org.junit.Assert.fail; 20 import static org.mockito.ArgumentMatchers.any; 21 import static org.mockito.ArgumentMatchers.anyString; 22 import static org.mockito.Mockito.doReturn; 23 import static org.mockito.Mockito.times; 24 import static org.mockito.Mockito.verify; 25 26 import com.android.regression.tests.MetricsXmlParser.ParseException; 27 import com.android.tradefed.build.BuildInfo; 28 import com.android.tradefed.config.OptionSetter; 29 import com.android.tradefed.invoker.IInvocationContext; 30 import com.android.tradefed.invoker.InvocationContext; 31 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 32 import com.android.tradefed.result.MetricsXMLResultReporter; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.util.proto.TfMetricProtoUtil; 35 36 import com.google.common.collect.ImmutableSet; 37 38 import org.junit.Before; 39 import org.junit.Test; 40 import org.junit.runner.RunWith; 41 import org.junit.runners.JUnit4; 42 import org.mockito.Mock; 43 import org.mockito.Mockito; 44 import org.mockito.MockitoAnnotations; 45 import org.mockito.Spy; 46 47 import java.io.ByteArrayInputStream; 48 import java.io.ByteArrayOutputStream; 49 import java.util.Collections; 50 import java.util.HashMap; 51 import java.util.Map; 52 import java.util.Set; 53 54 /** Simple unit tests for {@link MetricsXmlParser}. */ 55 @RunWith(JUnit4.class) 56 public class MetricsXmlParserTest { 57 58 @Spy private MetricsXMLResultReporter mResultReporter; 59 @Mock private Metrics mMetrics; 60 private ByteArrayOutputStream mOutputStream; 61 62 @Before setUp()63 public void setUp() throws Exception { 64 mOutputStream = new ByteArrayOutputStream(); 65 MockitoAnnotations.initMocks(this); 66 OptionSetter optionSetter = new OptionSetter(mResultReporter); 67 optionSetter.setOptionValue("metrics-folder", "/tmp"); 68 doReturn(mOutputStream).when(mResultReporter).createOutputStream(); 69 doReturn("ignore").when(mResultReporter).getTimeStamp(); 70 } 71 72 /** Test behavior when data to parse is empty */ 73 @Test testEmptyParse()74 public void testEmptyParse() { 75 try { 76 MetricsXmlParser.parse( 77 mMetrics, Collections.emptySet(), new ByteArrayInputStream(new byte[0])); 78 fail("ParseException not thrown"); 79 } catch (ParseException e) { 80 // expected 81 } 82 Mockito.verifyZeroInteractions(mMetrics); 83 } 84 85 /** Simple success test for xml parsing */ 86 @Test testSimpleParse()87 public void testSimpleParse() throws ParseException { 88 IInvocationContext context = new InvocationContext(); 89 context.addDeviceBuildInfo("fakeDevice", new BuildInfo()); 90 context.setTestTag("stub"); 91 mResultReporter.invocationStarted(context); 92 mResultReporter.testRunStarted("run", 3); 93 final TestDescription testId0 = new TestDescription("Test", "pass1"); 94 mResultReporter.testStarted(testId0); 95 mResultReporter.testEnded(testId0, new HashMap<String, Metric>()); 96 final TestDescription testId1 = new TestDescription("Test", "pass2"); 97 mResultReporter.testStarted(testId1); 98 mResultReporter.testEnded(testId1, new HashMap<String, Metric>()); 99 final TestDescription testId2 = new TestDescription("Test", "pass3"); 100 mResultReporter.testStarted(testId2); 101 mResultReporter.testEnded(testId2, new HashMap<String, Metric>()); 102 mResultReporter.testRunEnded(3, new HashMap<String, Metric>()); 103 mResultReporter.invocationEnded(5); 104 105 MetricsXmlParser.parse( 106 mMetrics, Collections.emptySet(), new ByteArrayInputStream(getOutput())); 107 verify(mMetrics).setNumTests(3); 108 verify(mMetrics).addRunMetric("time", "5"); 109 verify(mMetrics, times(0)).addTestMetric(any(), anyString(), anyString()); 110 Mockito.verifyNoMoreInteractions(mMetrics); 111 } 112 113 /** Test parsing a comprehensive document containing run metrics and test metrics */ 114 @Test testParse()115 public void testParse() throws ParseException { 116 IInvocationContext context = new InvocationContext(); 117 context.addDeviceBuildInfo("fakeDevice", new BuildInfo()); 118 context.setTestTag("stub"); 119 mResultReporter.invocationStarted(context); 120 mResultReporter.testRunStarted("run", 2); 121 122 final TestDescription testId0 = new TestDescription("Test", "pass1"); 123 mResultReporter.testStarted(testId0); 124 Map<String, String> testMetrics0 = new HashMap<>(); 125 testMetrics0.put("metric1", "1.1"); 126 mResultReporter.testEnded(testId0, TfMetricProtoUtil.upgradeConvert(testMetrics0)); 127 128 final TestDescription testId1 = new TestDescription("Test", "pass2"); 129 mResultReporter.testStarted(testId1); 130 Map<String, String> testMetrics1 = new HashMap<>(); 131 testMetrics1.put("metric2", "5.5"); 132 mResultReporter.testEnded(testId1, TfMetricProtoUtil.upgradeConvert(testMetrics1)); 133 134 Map<String, String> runMetrics = new HashMap<>(); 135 runMetrics.put("metric3", "8.8"); 136 mResultReporter.testRunEnded(3, TfMetricProtoUtil.upgradeConvert(runMetrics)); 137 mResultReporter.invocationEnded(5); 138 139 MetricsXmlParser.parse( 140 mMetrics, Collections.emptySet(), new ByteArrayInputStream(getOutput())); 141 142 verify(mMetrics).setNumTests(2); 143 verify(mMetrics).addRunMetric("metric3", "8.8"); 144 verify(mMetrics).addTestMetric(testId0, "metric1", "1.1"); 145 verify(mMetrics).addTestMetric(testId1, "metric2", "5.5"); 146 } 147 148 /** Test parsing a document with blacklist metrics */ 149 @Test testParseBlacklist()150 public void testParseBlacklist() throws ParseException { 151 IInvocationContext context = new InvocationContext(); 152 context.addDeviceBuildInfo("fakeDevice", new BuildInfo()); 153 context.setTestTag("stub"); 154 mResultReporter.invocationStarted(context); 155 mResultReporter.testRunStarted("run", 3); 156 157 final TestDescription testId0 = new TestDescription("Test", "pass1"); 158 mResultReporter.testStarted(testId0); 159 Map<String, String> testMetrics0 = new HashMap<>(); 160 testMetrics0.put("metric1", "1.1"); 161 mResultReporter.testEnded(testId0, TfMetricProtoUtil.upgradeConvert(testMetrics0)); 162 163 final TestDescription testId1 = new TestDescription("Test", "pass2"); 164 mResultReporter.testStarted(testId1); 165 Map<String, String> testMetrics1 = new HashMap<>(); 166 testMetrics1.put("metric2", "5.5"); 167 mResultReporter.testEnded(testId1, TfMetricProtoUtil.upgradeConvert(testMetrics1)); 168 169 Map<String, String> runMetrics = new HashMap<>(); 170 runMetrics.put("metric3", "8.8"); 171 mResultReporter.testRunEnded(3, TfMetricProtoUtil.upgradeConvert(runMetrics)); 172 mResultReporter.invocationEnded(5); 173 174 Set<String> blacklist = ImmutableSet.of("metric1", "metric3"); 175 176 MetricsXmlParser.parse(mMetrics, blacklist, new ByteArrayInputStream(getOutput())); 177 178 verify(mMetrics, times(0)).addRunMetric("metric3", "8.8"); 179 verify(mMetrics, times(0)).addTestMetric(testId0, "metric1", "1.1"); 180 verify(mMetrics).addTestMetric(testId1, "metric2", "5.5"); 181 } 182 183 /** Gets the output produced, stripping it of extraneous whitespace characters. */ getOutput()184 private byte[] getOutput() { 185 return mOutputStream.toByteArray(); 186 } 187 } 188